diff --git a/tests/regression/sgemm_tcore/kernel.cpp b/tests/regression/sgemm_tcore/kernel.cpp index a92d69ff..6b080369 100644 --- a/tests/regression/sgemm_tcore/kernel.cpp +++ b/tests/regression/sgemm_tcore/kernel.cpp @@ -3,545 +3,13 @@ #include #include #include "common.h" -#include "util.hpp" +#include "sgemm_impl.hpp" #include "include/gemmini.h" #include "gemmini_mmio.h" -#define GEMMINI_DMA 0 -#if SMEM_SIZE == 0x4000 -#define SMEM_ADDR_Q0 ((float * const) 0xff000000) -#define SMEM_ADDR_Q1 ((float * const) 0xff001000) -#define SMEM_ADDR_Q2 ((float * const) 0xff002000) -#define SMEM_ADDR_Q3 ((float * const) 0xff003000) -#define SPAD_ADDR_Q0 0x0 -#define SPAD_ADDR_Q1 0x80 -#define SPAD_ADDR_Q2 0x100 -#define SPAD_ADDR_Q3 0x180 -#define BOUND_INST 0x400040004ULL -#elif SMEM_SIZE == 0x10000 -#define SMEM_ADDR_Q0 ((float * const) 0xff000000) -#define SMEM_ADDR_Q1 ((float * const) 0xff004000) -#define SMEM_ADDR_Q2 ((float * const) 0xff008000) -#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) -#define SPAD_ADDR_Q0 0x0 -#define SPAD_ADDR_Q1 0x200 -#define SPAD_ADDR_Q2 0x400 -#define SPAD_ADDR_Q3 0x600 -#define BOUND_INST 0x800080008ULL -#else -#error Unsupported smem size -#endif - -// FIXME: NUM_THREADS and NUM_WARPS hardcoded -#if ((BM * BN / ELEM_PER_THREAD) > (CORES_PER_CLUSTER * 8 * 8)) -#error "threadblock size too big for cluster" -#endif - // using float_type = float; using float_type = float16_t; -// TODO: reduce args by passing leading A/B dimensions -template -inline void global_dmem_load(const uint32_t dim_m, const uint32_t dim_n, const uint32_t dim_k, - const uint32_t k, const T *A, const T *B, - volatile T *local_a, volatile T *local_b, - const uint32_t tid_in_threadblock, - const uint32_t threadblock_id_x, - const uint32_t threadblock_id_y) { - asm volatile ("global_dmem_load_start_%=:" :: ); - - // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do - // data movement at the fp32 granularity. Assuming that the matrix is stored - // row-major in GMEM, the packed fp16 pairs belong to the same row, - // neighboring columns; therefore, it essentially becomes equivalent to - // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed - // by a factor of two. - constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); - constexpr uint32_t BK_adjusted = BK / packed_factor; - const uint32_t dim_k_adjusted = dim_k / packed_factor; - constexpr uint32_t BN_adjusted = BN / packed_factor; - const uint32_t dim_n_adjusted = dim_n / packed_factor; - const uint32_t k_adjusted = k / packed_factor; - - const uint32_t local_a_row = tid_in_threadblock / BK_adjusted; - const uint32_t local_a_col = tid_in_threadblock % BK_adjusted; - const uint32_t local_as_row = tid_in_threadblock / BM; - const uint32_t local_as_col = tid_in_threadblock % BM; - const uint32_t local_b_row = tid_in_threadblock / BN_adjusted; - const uint32_t local_b_col = tid_in_threadblock % BN_adjusted; - - // FIXME: need fix for fp16? - constexpr uint32_t threads_in_threadblock = (BM * BN) / ELEM_PER_THREAD; - - // Data move from GMEM to SMEM - // - // Make sure global offset values for A and B are contiguous between - // neighboring threads to ensure GMEM coalescing. - // - // TODO: Sharedmem swizzling is important here - - // move A - if constexpr (!TRANSPOSE_AT_PRODUCE) { - // A is stored M-major in GMEM; - // no transpose at GMEM->SMEM movement - const uint32_t block_m = threadblock_id_y; - const uint32_t global_a_row = k_adjusted + local_as_row; - const uint32_t global_a_col = BM * block_m + local_as_col; - // number of rows a full TB can read at a time - // this is equivalent to threadblock_dim_y (assuming threadblock_dim_x == - // BK) - constexpr uint32_t row_stride_as = threads_in_threadblock / BM; - const float *global_a = reinterpret_cast(A) + - dim_m * global_a_row + global_a_col; - volatile float *local_a_tmp = reinterpret_cast(local_a) + - BM * local_as_row + local_as_col; - -#pragma GCC unroll 1 - for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; - local_row_offset += row_stride_as) { - // TODO: the code GCC generates for below seems fine atm, but unroll to - // assembly to be absolutely sure - *local_a_tmp = *global_a; - global_a += dim_m * row_stride_as; - local_a_tmp += BM * row_stride_as; - } - } else { - if constexpr (!GMEM_COALESCED_A) { - // !GMEM_COALESCED_A: threads do uncoalesced read from neighboring row in - // GMEM, writes to neighboring cols in SMEM - constexpr uint32_t row_stride_as = threads_in_threadblock / BM; - const uint32_t global_a_row = BM * threadblock_id_y + local_as_col; - const float *global_a = - reinterpret_cast(A) + dim_k_adjusted * global_a_row + (k_adjusted + local_as_row); - volatile float *local_a_tmp = - reinterpret_cast(local_a) + BM * local_as_row + local_as_col; - - static_assert( - row_stride_as * 8 <= BK_adjusted, - "manual loop unrolling condition not met; consider increasing BK"); - static_assert( - (BK_adjusted % (row_stride_as * 8)) == 0, - "manual loop unrolling condition not met; BK should be power-of-two"); - -#pragma GCC unroll 1 - for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; - local_row_offset += row_stride_as * 8) { - // @perf: bank conflicts here - // const uint32_t global_a_offset = - // dim_k_adjusted * (global_a_row) + (k + local_as_row + local_row_offset); - // local_a[BM * (local_as_row + local_row_offset) + local_as_col] = - // A[global_a_offset]; - - // *local_a_tmp = *global_a; - asm volatile ("flw ft0, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft1, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft2, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft3, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft4, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft5, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft6, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft7, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - - // NOTE: stride is fixed to word size , i.e. sizeof(float) = 4, - // regardless of fp16 or fp32. Since Vortex core does not support fp16, - // load things at word granularity and reinterpret bits inside the - // tensor core. - asm volatile ("fsw ft0, %0(%1)" :: "i"(BM * row_stride_as * 0 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft1, %0(%1)" :: "i"(BM * row_stride_as * 1 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft2, %0(%1)" :: "i"(BM * row_stride_as * 2 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft3, %0(%1)" :: "i"(BM * row_stride_as * 3 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft4, %0(%1)" :: "i"(BM * row_stride_as * 4 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(BM * row_stride_as * 5 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft6, %0(%1)" :: "i"(BM * row_stride_as * 6 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(BM * row_stride_as * 7 * sizeof(float)), "r"(local_a_tmp)); - local_a_tmp += BM * row_stride_as * 8; - } - } else { - constexpr uint32_t row_stride_a = threads_in_threadblock / BK_adjusted; - const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; - const float *global_a = reinterpret_cast(A) + - dim_k_adjusted * global_a_row + - (k_adjusted + local_a_col); - // NOTE that SMEM writes are transposed - volatile float *local_a_tmp = - reinterpret_cast(local_a) + BM * local_a_col + - local_a_row; - - static_assert( - row_stride_a * 8 <= BM, - "manual loop unrolling condition not met; consider increasing BM"); - static_assert( - (BM % (row_stride_a * 8)) == 0, - "manual loop unrolling condition not met; BM should be power-of-two"); - -#pragma GCC unroll 1 - for (uint32_t local_row_offset = 0; local_row_offset < BM; - local_row_offset += row_stride_a * 8) { - // const uint32_t global_a_offset = - // dim_k_adjusted * (global_a_row + local_row_offset) + (k + local_a_col); - // NOTE that SMEM writes are transposed - // local_a[BM * (local_a_col) + local_a_row + local_row_offset] = - // A[global_a_offset]; - - asm volatile ("flw ft0, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft1, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft2, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft3, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft4, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft5, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft6, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft7, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - - // stride along columns - asm volatile ("fsw ft0, %0(%1)" :: "i"(row_stride_a * 0 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft1, %0(%1)" :: "i"(row_stride_a * 1 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft2, %0(%1)" :: "i"(row_stride_a * 2 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft3, %0(%1)" :: "i"(row_stride_a * 3 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft4, %0(%1)" :: "i"(row_stride_a * 4 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(row_stride_a * 5 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft6, %0(%1)" :: "i"(row_stride_a * 6 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(row_stride_a * 7 * sizeof(float)), "r"(local_a_tmp)); - local_a_tmp += row_stride_a * 8; - } - } - } // end move A - - // move B - constexpr uint32_t row_stride_b = threads_in_threadblock / BN_adjusted; - const uint32_t global_b_col = BN_adjusted * threadblock_id_x + local_b_col; - // NOTE: not k_adjusted here; k is along the row dimension which is not - // compressed for fp16 - const float *global_b = reinterpret_cast(B) + - dim_n_adjusted * (k + local_b_row) + global_b_col; - volatile float *local_b_tmp = reinterpret_cast(local_b) + - BN_adjusted * local_b_row + local_b_col; - - static_assert( - row_stride_b * 8 <= BK_adjusted, - "manual loop unrolling condition not met; consider increasing BK"); - static_assert( - (BK_adjusted % (row_stride_b * 8)) == 0, - "manual loop unrolling condition not met; BK should be power-of-two"); - -#pragma GCC unroll 1 - for (uint32_t load_offset = 0; load_offset < BK; - load_offset += row_stride_b * 8) { - // equivalent code: - // - // *local_b_tmp = *global_b; - // global_b += dim_n * row_stride_b; - // local_b_tmp += BN * row_stride_b; - - asm volatile ("flw ft0, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft1, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft2, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft3, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft4, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft5, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft6, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft7, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - - asm volatile ("fsw ft0, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft1, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - asm volatile ("fsw ft2, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft3, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - asm volatile ("fsw ft4, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - asm volatile ("fsw ft6, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - } - - asm volatile ("global_dmem_load_finish_%=:" :: ); -} - -template -inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg, - const uint32_t tid_in_threadblock, - const uint32_t threads_per_threadblock, - const uint32_t threadblock_dim_y, - /*const uint32_t threadblock_id_x, - const uint32_t threadblock_id_y,*/ - const uint32_t threadblocks_per_cluster, - const uint32_t threadblock_id_in_cluster, - uint8_t *sharedmem_per_threadblock) { - const T *A = (const T *)arg->addr_a; - const T *B = (const T *)arg->addr_b; - float *C = (float *)arg->addr_c; - - const uint32_t dim_m = arg->dim_m; - const uint32_t dim_n = arg->dim_n; - const uint32_t dim_k = arg->dim_k; - - const uint32_t local_a_row = tid_in_threadblock / BK; - const uint32_t local_a_col = tid_in_threadblock % BK; - const uint32_t local_as_row = tid_in_threadblock / BM; - const uint32_t local_as_col = tid_in_threadblock % BM; - const uint32_t local_b_row = tid_in_threadblock / BN; - const uint32_t local_b_col = tid_in_threadblock % BN; - - // no double-buffering - const uint32_t threads_per_warpgroup = threads_per_threadblock; - const uint32_t warp_id_in_warpgroup = tid_in_threadblock / NUM_THREADS; - const uint32_t warp_row = warp_id_in_warpgroup / (BN / WN); - const uint32_t warp_col = warp_id_in_warpgroup % (BN / WN); - const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; - - volatile T *local_a = reinterpret_cast(sharedmem_per_threadblock); - constexpr size_t local_a_elems = (BM * BK); - volatile T *local_a_buf = local_a + local_a_elems; - - volatile T *local_b = local_a_buf + local_a_elems; - constexpr size_t local_b_elems = (BK * BN); - volatile T *local_b_buf = local_a_buf + local_b_elems; - - constexpr uint32_t skips = - loop_matmul_skips(/*skip_lda=*/0, /*skip_ldb=*/0, /*skip_ldd=*/1, - /*skip_ex=*/1, /*skip_stc=*/1); - -#if (GEMMINI_DMA == 1) - if (tid_in_threadblock == 0) { - gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); - // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, - // b_transpose); - - gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, - false, 0); - gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, - false, 1); - gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); - - gemmini_fence(); - } -#endif - - // divide rows (M) by the number of threadblocks - const uint32_t dim_m_range = (dim_m / threadblocks_per_cluster); - const uint32_t dim_m_start = dim_m_range * threadblock_id_in_cluster; - const uint32_t block_m_start = dim_m_start / BM; - const uint32_t block_m_end = (dim_m_start + dim_m_range) / BM; - -#pragma GCC unroll 1 - for (uint32_t block_m = block_m_start; block_m < block_m_end; block_m++) { -#pragma GCC unroll 1 - for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { - // clear out C - initialize_C(0); - initialize_C(1); - - if constexpr (GEMMINI_DMA) { - // pipeline initiation - if (tid_in_threadblock == 0) { - // configure dma gmem address to load from - // FIXME: block_k is wrong - ROCC_INSTRUCTION_RS1_RS2( - XCUSTOM_ACC, - (uint64_t)(A + block_m * BM * dim_k + /*block_k:*/0 * BK), - (uint64_t)(B + /*block_k:*/0 * BK * dim_n + block_n * BN), - k_LOOP_WS_CONFIG_ADDRS_AB) - // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB - GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); - gemmini_fence(); - - GEMMINI_CISC_CMD_I(10); - gemmini_fence(); - -#if 0 - // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS - // FIXME: block_k is 0 for two times - sp_tiled_matmul_full_spad_ws( -#if 1 - SPAD_ADDR_Q0, SPAD_ADDR_Q1, -#else - (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q2 : SPAD_ADDR_Q0, - (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q1, -#endif - /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q3, - /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, - /*pad_J=*/0, /*pad_K=*/0, - /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) - gemmini_fence(); -#endif - } - - threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); - } - -#pragma GCC unroll 1 - for (uint32_t block_k = 0; (block_k * BK) < dim_k; block_k++) { - - // producer code: GMEM->SMEM memory movement - // --------------------------------------------------------------------- - // - // this is either done using DMA or SIMT cores depending on GEMMINI_DMA - -#if (GEMMINI_DMA == 1) - if ((tid_in_threadblock == 0) && ((block_k * BK) != (dim_k - BK))) { - // configure dma gmem address to load from - // FIXME: block_k is wrong - ROCC_INSTRUCTION_RS1_RS2( - XCUSTOM_ACC, - (uint64_t)(A + block_m * BM * dim_k + (block_k + 1/*runahead*/) * BK), - (uint64_t)(B + (block_k + 1/*runahead*/) * BK * dim_n + block_n * BN), - k_LOOP_WS_CONFIG_ADDRS_AB) - // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB - GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); - // gemmini_fence(); - - // block_k is even: opcode 11 (write to local_a_buf) - // block_k is odd: opcode 10 (write to local_a) - const uint32_t opcode = 11 - (block_k & 1); - GEMMINI_CISC_CMD_R(opcode); - // // TODO: branch is probably slow - // if (block_k & 1) { - // GEMMINI_CISC_CMD_I(12); - // } else { // block_k == 0 is here - // GEMMINI_CISC_CMD_I(13); - // } - - // configure loop iteration bounds - // FIXME: shouldn't be necessary - // ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, - // k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, - // SPAD_ADDR_Q0, SPAD_ADDR_Q1, k_LOOP_WS_CONFIG_SPAD_AB) - // ROCC_INSTRUCTION_RS1_RS2( - // XCUSTOM_ACC, - // ((uint64_t)(/*a_spad_id:*/ 0) << 18) | - // ((uint64_t)(/*b_spad_id:*/ 0) << 16) | - // ((uint64_t)(/*act:0*/ 0) << 8) | ((/*low_D:*/ 0) << 2) | - // ((/*full_C:*/ 0) << 1) | (/*ex_accumulate:*/ 0), - // ((uint64_t)(/*C_spad_addr:*/ A) << 32) | 0x200U | (skips) | - // ((/*is_resadd*/ 0) << 2) | ((/*B_transpose:*/ 0) << 1) | - // (/*A_transpose:*/ 1), - // k_LOOP_WS) - // gemmini_fence(); - -#if 0 - uint32_t spad_a_produce; - uint32_t spad_b_produce; - const uint32_t mask_odd = (block_k & 1) << 31 >> 31; - const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; - spad_a_produce = - ((mask_odd & (SPAD_ADDR_Q0)) | (mask_even & (SPAD_ADDR_Q2))); - spad_b_produce = - ((mask_odd & (SPAD_ADDR_Q1)) | (mask_even & (SPAD_ADDR_Q3))); - // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS - // FIXME: block_k is 0 for two times - sp_tiled_matmul_full_spad_ws( - spad_a_produce, - spad_b_produce, - /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, - /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, - /*pad_J=*/0, /*pad_K=*/0, - /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) -#endif - } -#else - global_dmem_load(dim_m, dim_n, dim_k, block_k * BK, A, B, local_a, - local_b, tid_in_threadblock, block_n, block_m); - - threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); -#endif - - // consumer code: SMEM->RF and compute - // ---------------------------------------------------------------------- - // @perf: this loop spills to stack a lot because of all the flws in - const volatile T *local_a_consume; - const volatile T *local_b_consume; - if constexpr (GEMMINI_DMA) { - // local_a_consume = (k_index % 2) ? local_a_buf : local_a; - // local_b_consume = (k_index % 2) ? local_b_buf : local_b; - // FIXME: swap multiply with bitshifts - // const uint32_t mask_odd = (block_k & 1) << 31 >> 31; - // const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; - // local_a_consume = reinterpret_cast( - // (mask_odd & reinterpret_cast(local_a_buf)) | - // (mask_even & reinterpret_cast(local_a))); - // local_b_consume = reinterpret_cast( - // (mask_odd & reinterpret_cast(local_b_buf)) | - // (mask_even & reinterpret_cast(local_b))); - local_a_consume = local_a + (block_k & 1) * (local_a_elems); - local_b_consume = local_b + (block_k & 1) * (local_b_elems); - } else { - // no double-buffering without DMA - local_a_consume = local_a; - local_b_consume = local_b; - } - -#pragma GCC unroll 1 - for (int i = 0; i < BK_LOOP; i++) { -#pragma GCC unroll 4 - for (uint32_t local_k = 0; local_k < BK; local_k += TCK) { -#pragma GCC unroll 2 - for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { - // SMEM -> RF - vx_wmma_load_b(local_b_consume, local_k, warp_col, wn_iter, - tid_in_warp); -#pragma GCC unroll 2 - for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { - // SMEM -> RF - vx_wmma_load_a(local_a_consume, local_k, warp_row, wm_iter, - tid_in_warp); - // perform mma - vx_wmma(wm_iter); - } - } - } - } - - if constexpr (GEMMINI_DMA) { - // Call gemmini fence at the end of the loop to overlap dma & wmma. - // Hopefully by this time, dma would have finished so that this is a - // no-op - if (tid_in_threadblock == 0) { - gemmini_fence(); - } - } - - threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); - } - -#pragma GCC unroll 2 - for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { -#pragma GCC unroll 2 - for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { - write_results(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, - dim_n, C, block_n, block_m); - } - } - } - } -} - void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { // @perf: All threads are running these compute whose result is mostly same // across the threadblock diff --git a/tests/regression/sgemm_tcore/sgemm_impl.hpp b/tests/regression/sgemm_tcore/sgemm_impl.hpp new file mode 100644 index 00000000..9591228c --- /dev/null +++ b/tests/regression/sgemm_tcore/sgemm_impl.hpp @@ -0,0 +1,890 @@ +#ifndef _SGEMM_IMPL_H_ +#define _SGEMM_IMPL_H_ + +#include +#include +#include "include/gemmini.h" +#include "gemmini_mmio.h" + +// Constraints on parameters: +// * Memory: +// (BM + BN) * BK * sizeof(T) <= sharedmem size. +// BM * BK == BN * BK >= threadblock size >= NT * CORES_PER_CLUSTER +// When larger, the kernel runs a sequential loop to read into sharedmem; +// but smaller case is not handled. +// * Compute: +// ( M* N) / (TM*TN) == grid size >= NC*NW*NT +// (BM*BN) / (TM*TN) == threadblock size < NT * NW * CORES_PER_CLUSTER +// (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER +// * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields +// BM <= BK*TM*TN +#define BM 64 +#define BN 64 +#define BK 128 +#define WM 16 +#define WN 8 +#define TCM 8 +#define TCN 8 +#define TCK 16 +#define WMITER (WM / TCM) +#define WNITER (WN / TCN) +#define ELEM_PER_THREAD (WMITER * WNITER * (TCM * TCN) / NUM_THREADS) +// FIXME: NUM_THREADS and NUM_WARPS hardcoded +#if ((BM * BN / ELEM_PER_THREAD) > (CORES_PER_CLUSTER * 8 * 8)) +#error "threadblock size too big for cluster" +#endif + +// number of loop around the inner 0..TCK..BK loop to simulate perfect-DRAM +// scenario +#define BK_LOOP 1 +// Whether to transpose smem A tile at GMEM->SMEM (produce), or SMEM->RF +// (consume). This is because the tensor core expects the A tile to be stored +// in column-major order in SMEM, whereas it will be ultimately stored in +// row-major in the RF. +// +// For correctness, only one of either should be 1. E.g., PRODUCE 1 CONSUME 0 +// generates the NN kernel where both A and B are stored row-major in GMEM. +// To model the case where the A matrix is already stored column-major in GMEM, +// set both to 0. +#define TRANSPOSE_AT_PRODUCE 0 +#define TRANSPOSE_AT_CONSUME 0 +// GMEM_COALESCED: When TRANSPOSE_AT_PRODUCE == 1 (i.e. transpose at +// GMEM->SMEM), determines whether we do bank-conflict-free accesses for +// 1: GMEM loads of A matrix, or +// 0: SMEM stores of A matrix. +// +// Usually, GMEM_COALESCED==1 yields better performance since the memory +// behavior of GMEM is more sensitive to bank conflicts. +#define GMEM_COALESCED_A 1 + +#define GEMMINI_DMA 0 +#if SMEM_SIZE == 0x4000 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff001000) +#define SMEM_ADDR_Q2 ((float * const) 0xff002000) +#define SMEM_ADDR_Q3 ((float * const) 0xff003000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x80 +#define SPAD_ADDR_Q2 0x100 +#define SPAD_ADDR_Q3 0x180 +#define BOUND_INST 0x400040004ULL +#elif SMEM_SIZE == 0x10000 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff004000) +#define SMEM_ADDR_Q2 ((float * const) 0xff008000) +#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x200 +#define SPAD_ADDR_Q2 0x400 +#define SPAD_ADDR_Q3 0x600 +#define BOUND_INST 0x800080008ULL +#else +#error Unsupported smem size +#endif + +// "fake" fp16 type that only has the correct data width. +using float16_t = uint16_t; + +inline constexpr void map_operand_32lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // A (row major) + // Figure 7(a) in paper + // row 0~ 3: threadgroups 0 and 2 + // row 4~ 7: threadgroups 4 and 6 + // row 8~11: threadgroups 1 and 3 + // row 12~15: threadgroups 5 and 7 + row = tid % 4; + row += (tg * 8) % 16; + row += (tg / 4) * 4; + + // B (column major) + // NOTE: Matrix B mapping in Figure 7(a) is incorrect; below is the + // corrected mapping: + // col 0~ 3: threadgroups 0 and 1 + // col 4~ 7: threadgroups 4 and 5 + // col 8~11: threadgroups 2 and 3 + // col 12~15: threadgroups 6 and 7 + col = tid % 4; + col += ((tg % 4) / 2) * 8; + col += (tg / 4) * 4; +} + +inline constexpr void map_operand_8lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // A (row major) + // row 0~ 3: threadgroup 0 + // row 4~ 7: threadgroup 1 + row = tid % 4; + row += tg * 4; + + // B (column major) + // col 0~ 3: threadgroup 0 + // col 4~ 7: threadgroup 1 + col = tid % 4; + col += tg * 4; +} + +inline constexpr void map_operand(const int tid, int &row, int &col) { + if constexpr (NUM_THREADS == 32) { + map_operand_32lanes(tid, row, col); + } else if constexpr (NUM_THREADS == 8) { + map_operand_8lanes(tid, row, col); + } else { + // FIXME: not allowed + } +} + +inline constexpr void map_c_32lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // C + // Figure 7(b), left + col = ((tg % 4) / 2) * 8; + row = (tg * 8) % 16; + row += (tg / 4) * 4; + + // Figure 7(b), right + row += (tid % 4) % 2; + col += ((tid % 4) / 2) * 2; +} + +inline constexpr void map_c_8lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // C + col = 0; + row = tg * 4; + + // Figure 7(b), right + row += (tid % 4) % 2; + col += ((tid % 4) / 2) * 2; +} + +inline constexpr void map_c(const int tid, int &row, int &col) { + if constexpr (NUM_THREADS == 32) { + map_c_32lanes(tid, row, col); + } else if constexpr (NUM_THREADS == 8) { + map_c_8lanes(tid, row, col); + } else { + // FIXME: not allowed + } +} + +#define RISCV_CUSTOM3 0x7B + +inline void vx_wmma(const int dest_reg) { + if (dest_reg == 0) { + asm volatile (".insn r %0, 0, 0, x0, x0, x0" :: "i"(RISCV_CUSTOM3)); + } else { + asm volatile (".insn r %0, 0, 0, x1, x0, x0" :: "i"(RISCV_CUSTOM3)); + } +} + +// `local_k` is assumed to be multiple of TCK +template +inline void vx_wmma_load_a(volatile const T *smem_A, const int local_k, + const int warp_row, const int wm_iter, + const int thread_in_warp) { + asm volatile ("vx_wmma_load_a_start_%=:" :: ); + + const int tid = thread_in_warp; + const int tg = tid / 4; + + // @perf: this is duplicately computed in vx_wmma_load_a and vx_wmma_load_b + int row = 0; + int col = 0; + map_operand(tid, row, col); + + // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do + // data movement at the fp32 granularity. Assuming that the matrix is stored + // row-major in GMEM, the packed fp16 pairs belong to the same row, + // neighboring columns; therefore, it essentially becomes equivalent to + // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed + // by a factor of two. + constexpr int packed_factor = (std::is_same_v ? 2 : 1); + constexpr int BK_adjusted = BK / packed_factor; + const int local_k_adjusted = local_k / packed_factor; + + if constexpr (TRANSPOSE_AT_CONSUME) { + // A is stored K-major in smem + constexpr int smem_A_rows = BM; + constexpr int smem_A_cols = BK_adjusted; + + // int A_offset = (WM * warp_row + TCM * wm_iter + row) * smem_A_cols; + + // @perf: bank conflicts + // f8-f15 stores a single row of A + const volatile uint8_t *smem_addr; + smem_addr = reinterpret_cast( + &reinterpret_cast( + smem_A)[(WM * warp_row + TCM * wm_iter + row) * smem_A_cols + + local_k /* FIXME: adjust for fp16? */]); + // step to the next column + // threads read from different rows; bank conflicts + asm volatile("flw f0, %0(%1)" ::"i"(0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f1, %0(%1)" ::"i"(1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f2, %0(%1)" ::"i"(2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f3, %0(%1)" ::"i"(3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f4, %0(%1)" ::"i"(4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f5, %0(%1)" ::"i"(5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f6, %0(%1)" ::"i"(6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f7, %0(%1)" ::"i"(7 * sizeof(float)), "r"(smem_addr)); + } else { + // A is stored M-major in smem + constexpr int smem_AS_rows = BK_adjusted; + constexpr int smem_AS_cols = BM; + + const volatile uint8_t *smem_addr; + smem_addr = reinterpret_cast( + &reinterpret_cast( + smem_A)[((local_k_adjusted + 0) * smem_AS_cols) + + (WM * warp_row + TCM * wm_iter) + row]); + // f8-f15 stores a single row of A + // threads read from different columns; no bank conflicts + asm volatile("flw f0, %0(%1)" :: "i"(smem_AS_cols * 0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f1, %0(%1)" :: "i"(smem_AS_cols * 1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f2, %0(%1)" :: "i"(smem_AS_cols * 2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f3, %0(%1)" :: "i"(smem_AS_cols * 3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f4, %0(%1)" :: "i"(smem_AS_cols * 4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f5, %0(%1)" :: "i"(smem_AS_cols * 5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f6, %0(%1)" :: "i"(smem_AS_cols * 6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f7, %0(%1)" :: "i"(smem_AS_cols * 7 * sizeof(float)), "r"(smem_addr)); + } + + asm volatile ("vx_wmma_load_a_finish_%=:" :: ); +} + +// `local_k` is assumed to be multiple of TCK +template +inline void vx_wmma_load_b(const volatile T *smem_B, const int local_k, + const int warp_col, const int wn_iter, + const int thread_in_warp) { + asm volatile ("vx_wmma_load_b_start_%=:" :: ); + + const int tid = thread_in_warp; + const int tg = tid / 4; + + int row = 0; + int col = 0; + map_operand(tid, row, col); + + // see comment in vx_wmma_load_a + constexpr int packed_factor = (std::is_same_v ? 2 : 1); + constexpr int BK_adjusted = BN / packed_factor; + constexpr int BN_adjusted = BN / packed_factor; + const int local_k_adjusted = local_k / packed_factor; + + // B is stored N-major in smem + constexpr int smem_B_rows = BK_adjusted; + constexpr int smem_B_cols = BN; + + const volatile uint8_t *smem_addr; + smem_addr = reinterpret_cast( + &reinterpret_cast( + smem_B)[((local_k_adjusted + 0) * smem_B_cols) + + (WN * warp_col + TCN * wn_iter) + col]); + // f8-f15 stores a single column of B + // threads read from different columns; no bank conflicts + asm volatile("flw f8, %0(%1)" :: "i"(smem_B_cols * 0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f9, %0(%1)" :: "i"(smem_B_cols * 1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f10, %0(%1)" :: "i"(smem_B_cols * 2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f11, %0(%1)" :: "i"(smem_B_cols * 3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f12, %0(%1)" :: "i"(smem_B_cols * 4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f13, %0(%1)" :: "i"(smem_B_cols * 5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f14, %0(%1)" :: "i"(smem_B_cols * 6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f15, %0(%1)" :: "i"(smem_B_cols * 7 * sizeof(float)), "r"(smem_addr)); + + asm volatile ("vx_wmma_load_b_finish_%=:" :: ); +} + +inline void initialize_C(const int dest_reg) { + // initialize C to zeros + if (dest_reg == 0) { + asm volatile("fmv.w.x f16, x0"); + asm volatile("fmv.w.x f17, x0"); + asm volatile("fmv.w.x f18, x0"); + asm volatile("fmv.w.x f19, x0"); + asm volatile("fmv.w.x f20, x0"); + asm volatile("fmv.w.x f21, x0"); + asm volatile("fmv.w.x f22, x0"); + asm volatile("fmv.w.x f23, x0"); + } else { + asm volatile("fmv.w.x f24, x0"); + asm volatile("fmv.w.x f25, x0"); + asm volatile("fmv.w.x f26, x0"); + asm volatile("fmv.w.x f27, x0"); + asm volatile("fmv.w.x f28, x0"); + asm volatile("fmv.w.x f29, x0"); + asm volatile("fmv.w.x f30, x0"); + asm volatile("fmv.w.x f31, x0"); + } +} + +inline void write_results(const int thread_in_warp, const int warp_col, + const int warp_row, const int wn_iter, + const int wm_iter, const int dim_n, + float *C, const int threadblock_id_x, + const int threadblock_id_y) { + asm volatile ("write_results_start_%=:" :: ); + + int tid = thread_in_warp; + + // these are [0, TCM/TCN) + int tid_row = 0; + int tid_col = 0; + map_c(tid, tid_row, tid_col); + + int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; + int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; + + float *global_offset_C = + C + (BM * threadblock_id_y) * dim_n + BN * threadblock_id_x; + + // @perf: this likely causes a lot of gmem bank conflicts + if (wm_iter == 0) { + volatile uint8_t *gmem_addr = reinterpret_cast( + &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); + volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); + asm volatile ("fsw f16, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f17, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f18, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); + asm volatile ("fsw f19, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); + asm volatile ("fsw f20, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f21, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f22, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); + asm volatile ("fsw f23, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); + // asm volatile ("fsw f16, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 0)])); + // asm volatile ("fsw f17, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 1)])); + // asm volatile ("fsw f18, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 0)])); + // asm volatile ("fsw f19, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 1)])); + // asm volatile ("fsw f20, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 4)])); + // asm volatile ("fsw f21, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 5)])); + // asm volatile ("fsw f22, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 4)])); + // asm volatile ("fsw f23, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 5)])); + } else { + volatile uint8_t *gmem_addr = reinterpret_cast( + &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); + volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); + asm volatile ("fsw f24, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f25, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f26, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); + asm volatile ("fsw f27, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); + asm volatile ("fsw f28, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f29, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); + asm volatile ("fsw f30, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); + asm volatile ("fsw f31, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); + } + + asm volatile ("write_results_finish_%=:" :: ); +} + +inline void threadblock_barrier(const uint32_t barrier_id, const uint32_t count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +// TODO: reduce args by passing leading A/B dimensions +template +inline void global_dmem_load(const uint32_t dim_m, const uint32_t dim_n, const uint32_t dim_k, + const uint32_t k, const T *A, const T *B, + volatile T *local_a, volatile T *local_b, + const uint32_t tid_in_threadblock, + const uint32_t threadblock_id_x, + const uint32_t threadblock_id_y) { + asm volatile ("global_dmem_load_start_%=:" :: ); + + // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do + // data movement at the fp32 granularity. Assuming that the matrix is stored + // row-major in GMEM, the packed fp16 pairs belong to the same row, + // neighboring columns; therefore, it essentially becomes equivalent to + // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed + // by a factor of two. + constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); + constexpr uint32_t BK_adjusted = BK / packed_factor; + const uint32_t dim_k_adjusted = dim_k / packed_factor; + constexpr uint32_t BN_adjusted = BN / packed_factor; + const uint32_t dim_n_adjusted = dim_n / packed_factor; + const uint32_t k_adjusted = k / packed_factor; + + const uint32_t local_a_row = tid_in_threadblock / BK_adjusted; + const uint32_t local_a_col = tid_in_threadblock % BK_adjusted; + const uint32_t local_as_row = tid_in_threadblock / BM; + const uint32_t local_as_col = tid_in_threadblock % BM; + const uint32_t local_b_row = tid_in_threadblock / BN_adjusted; + const uint32_t local_b_col = tid_in_threadblock % BN_adjusted; + + // FIXME: need fix for fp16? + constexpr uint32_t threads_in_threadblock = (BM * BN) / ELEM_PER_THREAD; + + // Data move from GMEM to SMEM + // + // Make sure global offset values for A and B are contiguous between + // neighboring threads to ensure GMEM coalescing. + // + // TODO: Sharedmem swizzling is important here + + // move A + if constexpr (!TRANSPOSE_AT_PRODUCE) { + // A is stored M-major in GMEM; + // no transpose at GMEM->SMEM movement + const uint32_t block_m = threadblock_id_y; + const uint32_t global_a_row = k_adjusted + local_as_row; + const uint32_t global_a_col = BM * block_m + local_as_col; + // number of rows a full TB can read at a time + // this is equivalent to threadblock_dim_y (assuming threadblock_dim_x == + // BK) + constexpr uint32_t row_stride_as = threads_in_threadblock / BM; + const float *global_a = reinterpret_cast(A) + + dim_m * global_a_row + global_a_col; + volatile float *local_a_tmp = reinterpret_cast(local_a) + + BM * local_as_row + local_as_col; + +#pragma GCC unroll 1 + for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; + local_row_offset += row_stride_as) { + // TODO: the code GCC generates for below seems fine atm, but unroll to + // assembly to be absolutely sure + *local_a_tmp = *global_a; + global_a += dim_m * row_stride_as; + local_a_tmp += BM * row_stride_as; + } + } else { + if constexpr (!GMEM_COALESCED_A) { + // !GMEM_COALESCED_A: threads do uncoalesced read from neighboring row in + // GMEM, writes to neighboring cols in SMEM + constexpr uint32_t row_stride_as = threads_in_threadblock / BM; + const uint32_t global_a_row = BM * threadblock_id_y + local_as_col; + const float *global_a = + reinterpret_cast(A) + dim_k_adjusted * global_a_row + (k_adjusted + local_as_row); + volatile float *local_a_tmp = + reinterpret_cast(local_a) + BM * local_as_row + local_as_col; + + static_assert( + row_stride_as * 8 <= BK_adjusted, + "manual loop unrolling condition not met; consider increasing BK"); + static_assert( + (BK_adjusted % (row_stride_as * 8)) == 0, + "manual loop unrolling condition not met; BK should be power-of-two"); + +#pragma GCC unroll 1 + for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; + local_row_offset += row_stride_as * 8) { + // @perf: bank conflicts here + // const uint32_t global_a_offset = + // dim_k_adjusted * (global_a_row) + (k + local_as_row + local_row_offset); + // local_a[BM * (local_as_row + local_row_offset) + local_as_col] = + // A[global_a_offset]; + + // *local_a_tmp = *global_a; + asm volatile ("flw ft0, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft1, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft2, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft3, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft4, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft5, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft6, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + asm volatile ("flw ft7, (%0)" :: "r"(global_a)); + global_a += row_stride_as; + + // NOTE: stride is fixed to word size , i.e. sizeof(float) = 4, + // regardless of fp16 or fp32. Since Vortex core does not support fp16, + // load things at word granularity and reinterpret bits inside the + // tensor core. + asm volatile ("fsw ft0, %0(%1)" :: "i"(BM * row_stride_as * 0 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft1, %0(%1)" :: "i"(BM * row_stride_as * 1 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft2, %0(%1)" :: "i"(BM * row_stride_as * 2 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft3, %0(%1)" :: "i"(BM * row_stride_as * 3 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft4, %0(%1)" :: "i"(BM * row_stride_as * 4 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft5, %0(%1)" :: "i"(BM * row_stride_as * 5 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft6, %0(%1)" :: "i"(BM * row_stride_as * 6 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft7, %0(%1)" :: "i"(BM * row_stride_as * 7 * sizeof(float)), "r"(local_a_tmp)); + local_a_tmp += BM * row_stride_as * 8; + } + } else { + constexpr uint32_t row_stride_a = threads_in_threadblock / BK_adjusted; + const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; + const float *global_a = reinterpret_cast(A) + + dim_k_adjusted * global_a_row + + (k_adjusted + local_a_col); + // NOTE that SMEM writes are transposed + volatile float *local_a_tmp = + reinterpret_cast(local_a) + BM * local_a_col + + local_a_row; + + static_assert( + row_stride_a * 8 <= BM, + "manual loop unrolling condition not met; consider increasing BM"); + static_assert( + (BM % (row_stride_a * 8)) == 0, + "manual loop unrolling condition not met; BM should be power-of-two"); + +#pragma GCC unroll 1 + for (uint32_t local_row_offset = 0; local_row_offset < BM; + local_row_offset += row_stride_a * 8) { + // const uint32_t global_a_offset = + // dim_k_adjusted * (global_a_row + local_row_offset) + (k + local_a_col); + // NOTE that SMEM writes are transposed + // local_a[BM * (local_a_col) + local_a_row + local_row_offset] = + // A[global_a_offset]; + + asm volatile ("flw ft0, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft1, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft2, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft3, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft4, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft5, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft6, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + asm volatile ("flw ft7, (%0)" :: "r"(global_a)); + global_a += dim_k_adjusted * row_stride_a; + + // stride along columns + asm volatile ("fsw ft0, %0(%1)" :: "i"(row_stride_a * 0 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft1, %0(%1)" :: "i"(row_stride_a * 1 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft2, %0(%1)" :: "i"(row_stride_a * 2 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft3, %0(%1)" :: "i"(row_stride_a * 3 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft4, %0(%1)" :: "i"(row_stride_a * 4 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft5, %0(%1)" :: "i"(row_stride_a * 5 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft6, %0(%1)" :: "i"(row_stride_a * 6 * sizeof(float)), "r"(local_a_tmp)); + asm volatile ("fsw ft7, %0(%1)" :: "i"(row_stride_a * 7 * sizeof(float)), "r"(local_a_tmp)); + local_a_tmp += row_stride_a * 8; + } + } + } // end move A + + // move B + constexpr uint32_t row_stride_b = threads_in_threadblock / BN_adjusted; + const uint32_t global_b_col = BN_adjusted * threadblock_id_x + local_b_col; + // NOTE: not k_adjusted here; k is along the row dimension which is not + // compressed for fp16 + const float *global_b = reinterpret_cast(B) + + dim_n_adjusted * (k + local_b_row) + global_b_col; + volatile float *local_b_tmp = reinterpret_cast(local_b) + + BN_adjusted * local_b_row + local_b_col; + + static_assert( + row_stride_b * 8 <= BK_adjusted, + "manual loop unrolling condition not met; consider increasing BK"); + static_assert( + (BK_adjusted % (row_stride_b * 8)) == 0, + "manual loop unrolling condition not met; BK should be power-of-two"); + +#pragma GCC unroll 1 + for (uint32_t load_offset = 0; load_offset < BK; + load_offset += row_stride_b * 8) { + // equivalent code: + // + // *local_b_tmp = *global_b; + // global_b += dim_n * row_stride_b; + // local_b_tmp += BN * row_stride_b; + + asm volatile ("flw ft0, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft1, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft2, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft3, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft4, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft5, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft6, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + asm volatile ("flw ft7, (%0)" :: "r"(global_b)); + global_b += dim_n_adjusted * row_stride_b; + + asm volatile ("fsw ft0, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft1, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); + local_b_tmp += BN_adjusted * row_stride_b * 2; + asm volatile ("fsw ft2, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft3, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); + local_b_tmp += BN_adjusted * row_stride_b * 2; + asm volatile ("fsw ft4, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft5, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); + local_b_tmp += BN_adjusted * row_stride_b * 2; + asm volatile ("fsw ft6, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft7, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); + local_b_tmp += BN_adjusted * row_stride_b * 2; + } + + asm volatile ("global_dmem_load_finish_%=:" :: ); +} + +template +inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg, + const uint32_t tid_in_threadblock, + const uint32_t threads_per_threadblock, + const uint32_t threadblock_dim_y, + /*const uint32_t threadblock_id_x, + const uint32_t threadblock_id_y,*/ + const uint32_t threadblocks_per_cluster, + const uint32_t threadblock_id_in_cluster, + uint8_t *sharedmem_per_threadblock) { + const T *A = (const T *)arg->addr_a; + const T *B = (const T *)arg->addr_b; + float *C = (float *)arg->addr_c; + + const uint32_t dim_m = arg->dim_m; + const uint32_t dim_n = arg->dim_n; + const uint32_t dim_k = arg->dim_k; + + const uint32_t local_a_row = tid_in_threadblock / BK; + const uint32_t local_a_col = tid_in_threadblock % BK; + const uint32_t local_as_row = tid_in_threadblock / BM; + const uint32_t local_as_col = tid_in_threadblock % BM; + const uint32_t local_b_row = tid_in_threadblock / BN; + const uint32_t local_b_col = tid_in_threadblock % BN; + + // no double-buffering + const uint32_t threads_per_warpgroup = threads_per_threadblock; + const uint32_t warp_id_in_warpgroup = tid_in_threadblock / NUM_THREADS; + const uint32_t warp_row = warp_id_in_warpgroup / (BN / WN); + const uint32_t warp_col = warp_id_in_warpgroup % (BN / WN); + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + + volatile T *local_a = reinterpret_cast(sharedmem_per_threadblock); + constexpr size_t local_a_elems = (BM * BK); + volatile T *local_a_buf = local_a + local_a_elems; + + volatile T *local_b = local_a_buf + local_a_elems; + constexpr size_t local_b_elems = (BK * BN); + volatile T *local_b_buf = local_a_buf + local_b_elems; + + constexpr uint32_t skips = + loop_matmul_skips(/*skip_lda=*/0, /*skip_ldb=*/0, /*skip_ldd=*/1, + /*skip_ex=*/1, /*skip_stc=*/1); + +#if (GEMMINI_DMA == 1) + if (tid_in_threadblock == 0) { + gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, + // b_transpose); + + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, + false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, + false, 1); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + + gemmini_fence(); + } +#endif + + // divide rows (M) by the number of threadblocks + const uint32_t dim_m_range = (dim_m / threadblocks_per_cluster); + const uint32_t dim_m_start = dim_m_range * threadblock_id_in_cluster; + const uint32_t block_m_start = dim_m_start / BM; + const uint32_t block_m_end = (dim_m_start + dim_m_range) / BM; + +#pragma GCC unroll 1 + for (uint32_t block_m = block_m_start; block_m < block_m_end; block_m++) { +#pragma GCC unroll 1 + for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { + // clear out C + initialize_C(0); + initialize_C(1); + + if constexpr (GEMMINI_DMA) { + // pipeline initiation + if (tid_in_threadblock == 0) { + // configure dma gmem address to load from + // FIXME: block_k is wrong + ROCC_INSTRUCTION_RS1_RS2( + XCUSTOM_ACC, + (uint64_t)(A + block_m * BM * dim_k + /*block_k:*/0 * BK), + (uint64_t)(B + /*block_k:*/0 * BK * dim_n + block_n * BN), + k_LOOP_WS_CONFIG_ADDRS_AB) + // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB + GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); + gemmini_fence(); + + GEMMINI_CISC_CMD_I(10); + gemmini_fence(); + +#if 0 + // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS + // FIXME: block_k is 0 for two times + sp_tiled_matmul_full_spad_ws( +#if 1 + SPAD_ADDR_Q0, SPAD_ADDR_Q1, +#else + (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q2 : SPAD_ADDR_Q0, + (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q1, +#endif + /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q3, + /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, + /*pad_J=*/0, /*pad_K=*/0, + /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, + /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) + gemmini_fence(); +#endif + } + + threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); + } + +#pragma GCC unroll 1 + for (uint32_t block_k = 0; (block_k * BK) < dim_k; block_k++) { + + // producer code: GMEM->SMEM memory movement + // --------------------------------------------------------------------- + // + // this is either done using DMA or SIMT cores depending on GEMMINI_DMA + +#if (GEMMINI_DMA == 1) + if ((tid_in_threadblock == 0) && ((block_k * BK) != (dim_k - BK))) { + // configure dma gmem address to load from + // FIXME: block_k is wrong + ROCC_INSTRUCTION_RS1_RS2( + XCUSTOM_ACC, + (uint64_t)(A + block_m * BM * dim_k + (block_k + 1/*runahead*/) * BK), + (uint64_t)(B + (block_k + 1/*runahead*/) * BK * dim_n + block_n * BN), + k_LOOP_WS_CONFIG_ADDRS_AB) + // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB + GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); + // gemmini_fence(); + + // block_k is even: opcode 11 (write to local_a_buf) + // block_k is odd: opcode 10 (write to local_a) + const uint32_t opcode = 11 - (block_k & 1); + GEMMINI_CISC_CMD_R(opcode); + // // TODO: branch is probably slow + // if (block_k & 1) { + // GEMMINI_CISC_CMD_I(12); + // } else { // block_k == 0 is here + // GEMMINI_CISC_CMD_I(13); + // } + + // configure loop iteration bounds + // FIXME: shouldn't be necessary + // ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, + // k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, + // SPAD_ADDR_Q0, SPAD_ADDR_Q1, k_LOOP_WS_CONFIG_SPAD_AB) + // ROCC_INSTRUCTION_RS1_RS2( + // XCUSTOM_ACC, + // ((uint64_t)(/*a_spad_id:*/ 0) << 18) | + // ((uint64_t)(/*b_spad_id:*/ 0) << 16) | + // ((uint64_t)(/*act:0*/ 0) << 8) | ((/*low_D:*/ 0) << 2) | + // ((/*full_C:*/ 0) << 1) | (/*ex_accumulate:*/ 0), + // ((uint64_t)(/*C_spad_addr:*/ A) << 32) | 0x200U | (skips) | + // ((/*is_resadd*/ 0) << 2) | ((/*B_transpose:*/ 0) << 1) | + // (/*A_transpose:*/ 1), + // k_LOOP_WS) + // gemmini_fence(); + +#if 0 + uint32_t spad_a_produce; + uint32_t spad_b_produce; + const uint32_t mask_odd = (block_k & 1) << 31 >> 31; + const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; + spad_a_produce = + ((mask_odd & (SPAD_ADDR_Q0)) | (mask_even & (SPAD_ADDR_Q2))); + spad_b_produce = + ((mask_odd & (SPAD_ADDR_Q1)) | (mask_even & (SPAD_ADDR_Q3))); + // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS + // FIXME: block_k is 0 for two times + sp_tiled_matmul_full_spad_ws( + spad_a_produce, + spad_b_produce, + /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, + /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, + /*pad_J=*/0, /*pad_K=*/0, + /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, + /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) +#endif + } +#else + global_dmem_load(dim_m, dim_n, dim_k, block_k * BK, A, B, local_a, + local_b, tid_in_threadblock, block_n, block_m); + + threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); +#endif + + // consumer code: SMEM->RF and compute + // ---------------------------------------------------------------------- + // @perf: this loop spills to stack a lot because of all the flws in + const volatile T *local_a_consume; + const volatile T *local_b_consume; + if constexpr (GEMMINI_DMA) { + // local_a_consume = (k_index % 2) ? local_a_buf : local_a; + // local_b_consume = (k_index % 2) ? local_b_buf : local_b; + // FIXME: swap multiply with bitshifts + // const uint32_t mask_odd = (block_k & 1) << 31 >> 31; + // const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; + // local_a_consume = reinterpret_cast( + // (mask_odd & reinterpret_cast(local_a_buf)) | + // (mask_even & reinterpret_cast(local_a))); + // local_b_consume = reinterpret_cast( + // (mask_odd & reinterpret_cast(local_b_buf)) | + // (mask_even & reinterpret_cast(local_b))); + local_a_consume = local_a + (block_k & 1) * (local_a_elems); + local_b_consume = local_b + (block_k & 1) * (local_b_elems); + } else { + // no double-buffering without DMA + local_a_consume = local_a; + local_b_consume = local_b; + } + +#pragma GCC unroll 1 + for (int i = 0; i < BK_LOOP; i++) { +#pragma GCC unroll 4 + for (uint32_t local_k = 0; local_k < BK; local_k += TCK) { +#pragma GCC unroll 2 + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + // SMEM -> RF + vx_wmma_load_b(local_b_consume, local_k, warp_col, wn_iter, + tid_in_warp); +#pragma GCC unroll 2 + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { + // SMEM -> RF + vx_wmma_load_a(local_a_consume, local_k, warp_row, wm_iter, + tid_in_warp); + // perform mma + vx_wmma(wm_iter); + } + } + } + } + + if constexpr (GEMMINI_DMA) { + // Call gemmini fence at the end of the loop to overlap dma & wmma. + // Hopefully by this time, dma would have finished so that this is a + // no-op + if (tid_in_threadblock == 0) { + gemmini_fence(); + } + } + + threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); + } + +#pragma GCC unroll 2 + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { +#pragma GCC unroll 2 + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + write_results(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, + dim_n, C, block_n, block_m); + } + } + } + } +} + +#endif diff --git a/tests/regression/sgemm_tcore/util.hpp b/tests/regression/sgemm_tcore/util.hpp deleted file mode 100644 index 6f8772f1..00000000 --- a/tests/regression/sgemm_tcore/util.hpp +++ /dev/null @@ -1,359 +0,0 @@ -#ifndef _UTIL_H_ -#define _UTIL_H_ - -#include -#include -#include "include/gemmini.h" -#include "gemmini_mmio.h" - -// Constraints on parameters: -// * Memory: -// (BM + BN) * BK * sizeof(T) <= sharedmem size. -// BM * BK == BN * BK >= threadblock size >= NT * CORES_PER_CLUSTER -// When larger, the kernel runs a sequential loop to read into sharedmem; -// but smaller case is not handled. -// * Compute: -// ( M* N) / (TM*TN) == grid size >= NC*NW*NT -// (BM*BN) / (TM*TN) == threadblock size < NT * NW * CORES_PER_CLUSTER -// (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER -// * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields -// BM <= BK*TM*TN -#define BM 64 -#define BN 64 -#define BK 128 -#define WM 16 -#define WN 8 -#define TCM 8 -#define TCN 8 -#define TCK 16 -#define WMITER (WM / TCM) -#define WNITER (WN / TCN) -#define ELEM_PER_THREAD (WMITER * WNITER * (TCM * TCN) / NUM_THREADS) - -// number of loop around the inner 0..TCK..BK loop to simulate perfect-DRAM -// scenario -#define BK_LOOP 1 -// Whether to transpose smem A tile at GMEM->SMEM (produce), or SMEM->RF -// (consume). This is because the tensor core expects the A tile to be stored -// in column-major order in SMEM, whereas it will be ultimately stored in -// row-major in the RF. -// -// For correctness, only one of either should be 1. E.g., PRODUCE 1 CONSUME 0 -// generates the NN kernel where both A and B are stored row-major in GMEM. -// To model the case where the A matrix is already stored column-major in GMEM, -// set both to 0. -#define TRANSPOSE_AT_PRODUCE 0 -#define TRANSPOSE_AT_CONSUME 0 -// GMEM_COALESCED: When TRANSPOSE_AT_PRODUCE == 1 (i.e. transpose at -// GMEM->SMEM), determines whether we do bank-conflict-free accesses for -// 1: GMEM loads of A matrix, or -// 0: SMEM stores of A matrix. -// -// Usually, GMEM_COALESCED==1 yields better performance since the memory -// behavior of GMEM is more sensitive to bank conflicts. -#define GMEM_COALESCED_A 1 - -// "fake" fp16 type that only has the correct data width. -using float16_t = uint16_t; - -inline constexpr void map_operand_32lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // A (row major) - // Figure 7(a) in paper - // row 0~ 3: threadgroups 0 and 2 - // row 4~ 7: threadgroups 4 and 6 - // row 8~11: threadgroups 1 and 3 - // row 12~15: threadgroups 5 and 7 - row = tid % 4; - row += (tg * 8) % 16; - row += (tg / 4) * 4; - - // B (column major) - // NOTE: Matrix B mapping in Figure 7(a) is incorrect; below is the - // corrected mapping: - // col 0~ 3: threadgroups 0 and 1 - // col 4~ 7: threadgroups 4 and 5 - // col 8~11: threadgroups 2 and 3 - // col 12~15: threadgroups 6 and 7 - col = tid % 4; - col += ((tg % 4) / 2) * 8; - col += (tg / 4) * 4; -} - -inline constexpr void map_operand_8lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // A (row major) - // row 0~ 3: threadgroup 0 - // row 4~ 7: threadgroup 1 - row = tid % 4; - row += tg * 4; - - // B (column major) - // col 0~ 3: threadgroup 0 - // col 4~ 7: threadgroup 1 - col = tid % 4; - col += tg * 4; -} - -inline constexpr void map_operand(const int tid, int &row, int &col) { - if constexpr (NUM_THREADS == 32) { - map_operand_32lanes(tid, row, col); - } else if constexpr (NUM_THREADS == 8) { - map_operand_8lanes(tid, row, col); - } else { - // FIXME: not allowed - } -} - -inline constexpr void map_c_32lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // C - // Figure 7(b), left - col = ((tg % 4) / 2) * 8; - row = (tg * 8) % 16; - row += (tg / 4) * 4; - - // Figure 7(b), right - row += (tid % 4) % 2; - col += ((tid % 4) / 2) * 2; -} - -inline constexpr void map_c_8lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // C - col = 0; - row = tg * 4; - - // Figure 7(b), right - row += (tid % 4) % 2; - col += ((tid % 4) / 2) * 2; -} - -inline constexpr void map_c(const int tid, int &row, int &col) { - if constexpr (NUM_THREADS == 32) { - map_c_32lanes(tid, row, col); - } else if constexpr (NUM_THREADS == 8) { - map_c_8lanes(tid, row, col); - } else { - // FIXME: not allowed - } -} - -#define RISCV_CUSTOM3 0x7B - -inline void vx_wmma(const int dest_reg) { - if (dest_reg == 0) { - asm volatile (".insn r %0, 0, 0, x0, x0, x0" :: "i"(RISCV_CUSTOM3)); - } else { - asm volatile (".insn r %0, 0, 0, x1, x0, x0" :: "i"(RISCV_CUSTOM3)); - } -} - -// `local_k` is assumed to be multiple of TCK -template -inline void vx_wmma_load_a(volatile const T *smem_A, const int local_k, - const int warp_row, const int wm_iter, - const int thread_in_warp) { - asm volatile ("vx_wmma_load_a_start_%=:" :: ); - - const int tid = thread_in_warp; - const int tg = tid / 4; - - // @perf: this is duplicately computed in vx_wmma_load_a and vx_wmma_load_b - int row = 0; - int col = 0; - map_operand(tid, row, col); - - // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do - // data movement at the fp32 granularity. Assuming that the matrix is stored - // row-major in GMEM, the packed fp16 pairs belong to the same row, - // neighboring columns; therefore, it essentially becomes equivalent to - // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed - // by a factor of two. - constexpr int packed_factor = (std::is_same_v ? 2 : 1); - constexpr int BK_adjusted = BK / packed_factor; - const int local_k_adjusted = local_k / packed_factor; - - if constexpr (TRANSPOSE_AT_CONSUME) { - // A is stored K-major in smem - constexpr int smem_A_rows = BM; - constexpr int smem_A_cols = BK_adjusted; - - // int A_offset = (WM * warp_row + TCM * wm_iter + row) * smem_A_cols; - - // @perf: bank conflicts - // f8-f15 stores a single row of A - const volatile uint8_t *smem_addr; - smem_addr = reinterpret_cast( - &reinterpret_cast( - smem_A)[(WM * warp_row + TCM * wm_iter + row) * smem_A_cols + - local_k /* FIXME: adjust for fp16? */]); - // step to the next column - // threads read from different rows; bank conflicts - asm volatile("flw f0, %0(%1)" ::"i"(0 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f1, %0(%1)" ::"i"(1 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f2, %0(%1)" ::"i"(2 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f3, %0(%1)" ::"i"(3 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f4, %0(%1)" ::"i"(4 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f5, %0(%1)" ::"i"(5 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f6, %0(%1)" ::"i"(6 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f7, %0(%1)" ::"i"(7 * sizeof(float)), "r"(smem_addr)); - } else { - // A is stored M-major in smem - constexpr int smem_AS_rows = BK_adjusted; - constexpr int smem_AS_cols = BM; - - const volatile uint8_t *smem_addr; - smem_addr = reinterpret_cast( - &reinterpret_cast( - smem_A)[((local_k_adjusted + 0) * smem_AS_cols) + - (WM * warp_row + TCM * wm_iter) + row]); - // f8-f15 stores a single row of A - // threads read from different columns; no bank conflicts - asm volatile("flw f0, %0(%1)" :: "i"(smem_AS_cols * 0 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f1, %0(%1)" :: "i"(smem_AS_cols * 1 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f2, %0(%1)" :: "i"(smem_AS_cols * 2 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f3, %0(%1)" :: "i"(smem_AS_cols * 3 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f4, %0(%1)" :: "i"(smem_AS_cols * 4 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f5, %0(%1)" :: "i"(smem_AS_cols * 5 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f6, %0(%1)" :: "i"(smem_AS_cols * 6 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f7, %0(%1)" :: "i"(smem_AS_cols * 7 * sizeof(float)), "r"(smem_addr)); - } - - asm volatile ("vx_wmma_load_a_finish_%=:" :: ); -} - -// `local_k` is assumed to be multiple of TCK -template -inline void vx_wmma_load_b(const volatile T *smem_B, const int local_k, - const int warp_col, const int wn_iter, - const int thread_in_warp) { - asm volatile ("vx_wmma_load_b_start_%=:" :: ); - - const int tid = thread_in_warp; - const int tg = tid / 4; - - int row = 0; - int col = 0; - map_operand(tid, row, col); - - // see comment in vx_wmma_load_a - constexpr int packed_factor = (std::is_same_v ? 2 : 1); - constexpr int BK_adjusted = BN / packed_factor; - constexpr int BN_adjusted = BN / packed_factor; - const int local_k_adjusted = local_k / packed_factor; - - // B is stored N-major in smem - constexpr int smem_B_rows = BK_adjusted; - constexpr int smem_B_cols = BN; - - const volatile uint8_t *smem_addr; - smem_addr = reinterpret_cast( - &reinterpret_cast( - smem_B)[((local_k_adjusted + 0) * smem_B_cols) + - (WN * warp_col + TCN * wn_iter) + col]); - // f8-f15 stores a single column of B - // threads read from different columns; no bank conflicts - asm volatile("flw f8, %0(%1)" :: "i"(smem_B_cols * 0 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f9, %0(%1)" :: "i"(smem_B_cols * 1 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f10, %0(%1)" :: "i"(smem_B_cols * 2 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f11, %0(%1)" :: "i"(smem_B_cols * 3 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f12, %0(%1)" :: "i"(smem_B_cols * 4 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f13, %0(%1)" :: "i"(smem_B_cols * 5 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f14, %0(%1)" :: "i"(smem_B_cols * 6 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f15, %0(%1)" :: "i"(smem_B_cols * 7 * sizeof(float)), "r"(smem_addr)); - - asm volatile ("vx_wmma_load_b_finish_%=:" :: ); -} - -inline void initialize_C(const int dest_reg) { - // initialize C to zeros - if (dest_reg == 0) { - asm volatile("fmv.w.x f16, x0"); - asm volatile("fmv.w.x f17, x0"); - asm volatile("fmv.w.x f18, x0"); - asm volatile("fmv.w.x f19, x0"); - asm volatile("fmv.w.x f20, x0"); - asm volatile("fmv.w.x f21, x0"); - asm volatile("fmv.w.x f22, x0"); - asm volatile("fmv.w.x f23, x0"); - } else { - asm volatile("fmv.w.x f24, x0"); - asm volatile("fmv.w.x f25, x0"); - asm volatile("fmv.w.x f26, x0"); - asm volatile("fmv.w.x f27, x0"); - asm volatile("fmv.w.x f28, x0"); - asm volatile("fmv.w.x f29, x0"); - asm volatile("fmv.w.x f30, x0"); - asm volatile("fmv.w.x f31, x0"); - } -} - -inline void write_results(const int thread_in_warp, const int warp_col, - const int warp_row, const int wn_iter, - const int wm_iter, const int dim_n, - float *C, const int threadblock_id_x, - const int threadblock_id_y) { - asm volatile ("write_results_start_%=:" :: ); - - int tid = thread_in_warp; - - // these are [0, TCM/TCN) - int tid_row = 0; - int tid_col = 0; - map_c(tid, tid_row, tid_col); - - int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; - int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; - - float *global_offset_C = - C + (BM * threadblock_id_y) * dim_n + BN * threadblock_id_x; - - // @perf: this likely causes a lot of gmem bank conflicts - if (wm_iter == 0) { - volatile uint8_t *gmem_addr = reinterpret_cast( - &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); - volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); - asm volatile ("fsw f16, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f17, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f18, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f19, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f20, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f21, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f22, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f23, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); - // asm volatile ("fsw f16, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 0)])); - // asm volatile ("fsw f17, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 1)])); - // asm volatile ("fsw f18, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 0)])); - // asm volatile ("fsw f19, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 1)])); - // asm volatile ("fsw f20, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 4)])); - // asm volatile ("fsw f21, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 5)])); - // asm volatile ("fsw f22, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 4)])); - // asm volatile ("fsw f23, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 5)])); - } else { - volatile uint8_t *gmem_addr = reinterpret_cast( - &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); - volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); - asm volatile ("fsw f24, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f25, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f26, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f27, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f28, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f29, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f30, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f31, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); - } - - asm volatile ("write_results_finish_%=:" :: ); -} - -inline void threadblock_barrier(const uint32_t barrier_id, const uint32_t count) { - vx_fence(); - vx_barrier(barrier_id, count); -} - -#endif