Merge branch 'kernels' of https://github.com/hansungk/vortex-private into kernels

2024-04-01 11:09:43 -07:00
parent b88dbd7a83 b0c1f77388
commit e6db1a83af
15 changed files with 535 additions and 68 deletions
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1 +1,7 @@
 **/*.log
+.depend
+*.bin
+*.dump
+*.elf
+*.o
+*.ll
--- a/tests/opencl/convolution/main.cc
+++ b/tests/opencl/convolution/main.cc
@@ -238,9 +238,9 @@ int main (int argc, char **argv) {
  }

  // NOTE(hansung): Dump operand buffer to a file
-  if (write_operand_file("matmul.input.a.bin", h_a.data(), nbytes) != 0)
+  if (write_operand_file("convolution.input.input.bin", h_i.data(), i_nbytes) != 0)
    return EXIT_FAILURE;
-  if (write_operand_file("matmul.input.b.bin", h_b.data(), nbytes) != 0)
+  if (write_operand_file("convolution.input.weights.bin", h_w.data(), w_nbytes) != 0)
    return EXIT_FAILURE;

  // Creating command queue
--- a/tests/regression/common.mk
+++ b/tests/regression/common.mk
@@ -78,17 +78,23 @@ endif
 endif
 endif

-all: $(PROJECT) kernel.bin kernel.dump
+all: $(PROJECT) kernel.bin kernel.dump kernel.radiance.dump

 kernel.dump: kernel.elf
 	$(VX_DP) -D kernel.elf > kernel.dump

-kernel.bin: kernel.elf
+kernel.radiance.dump: kernel.radiance.elf
+	$(VX_DP) -D kernel.radiance.elf > kernel.radiance.dump
+
+kernel.bin: kernel.elf kernel.radiance.elf
 	$(VX_CP) -O binary kernel.elf kernel.bin

 kernel.elf: $(VX_SRCS)
 	$(VX_CXX) $(VX_CFLAGS) $(VX_SRCS) $(VX_LDFLAGS) -o kernel.elf

+kernel.radiance.elf: $(VX_SRCS)
+	$(VX_CXX) $(VX_CFLAGS) $(VX_SRCS) $(VX_LDFLAGS) -DRADIANCE -o kernel.radiance.elf
+
 $(PROJECT): $(SRCS)
 	$(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@

@@ -115,7 +121,7 @@ clean:
 	rm -rf $(PROJECT) *.o .depend

 clean-all: clean
-	rm -rf *.elf *.bin *.dump
+	rm -rf kernel.elf kernel.radiance.elf *.dump

 ifneq ($(MAKECMDGOALS),clean)
    -include .depend
--- a/tests/regression/sgemm_wg/Makefile
+++ b/tests/regression/sgemm_wg/Makefile
@@ -1,6 +1,6 @@
 PROJECT = sgemm_wg

-SRCS = main.cpp
+SRCS = main.cpp common.h

 VX_SRCS = kernel.cpp

--- a/tests/regression/sgemm_wg/kernel.cpp
+++ b/tests/regression/sgemm_wg/kernel.cpp
@@ -1,84 +1,192 @@
 #include <stdint.h>
 #include <vx_intrinsics.h>
+#include <vx_print.h>
 #include <vx_spawn.h>
 #include "common.h"

-inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg,
-                              const uint32_t tid_in_threadblock_x,
-                              const uint32_t tid_in_threadblock_y,
+// Constraints on parameters:
+// * Memory:
+//   (BM + BN) * BK * sizeof(float) <= sharedmem size.
+//   BM * BK == BN * BK >= threadblock size >= NT * CORES_PER_CLUSTER
+//     When larger, the kernel runs a sequential loop to read into sharedmem;
+//     but smaller case is not handled.
+// * Compute:
+//   ( M* N) / (TM*TN) == grid size >= NC*NW*NT
+//   (BM*BN) / (TM*TN) == threadblock size < NT * NW * CORES_PER_CLUSTER
+//   (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER
+// * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields
+//   BM <= BK*TM*TN
+#define BM 16
+#define BN BM
+#define BK 4
+#define TM 4
+#define TN 4
+
+void threadblock_barrier(unsigned int tid_in_threadblock, unsigned int barrier_id, unsigned int count) {
+    vx_fence();
+    vx_barrier(barrier_id, count);
+}
+
+void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg,
+                              const uint32_t tid_in_threadblock,
                              const uint32_t threadblock_dim_x,
                              const uint32_t threadblock_dim_y,
                              const uint32_t threadblock_id_x,
-                              const uint32_t threadblock_id_y) {
-  const float *global_a = (const float *)arg->addr_a;
-  const float *global_b = (const float *)arg->addr_b;
-  float *global_c = (float *)arg->addr_c;
+                              const uint32_t threadblock_id_y,
+                              const uint32_t threadblock_id_in_cluster,
+                              float *sharedmem_per_threadblock) {
+  const float *A = (const float *)arg->addr_a;
+  const float *B = (const float *)arg->addr_b;
+  float *C = (float *)arg->addr_c;

  // assumes NT == NW == matrix_dim
  const uint32_t dim_m = arg->dim_m;
  const uint32_t dim_n = arg->dim_n;
  const uint32_t dim_k = arg->dim_k;

-  // FIXME: assumes local block size is square shape
-  const uint32_t local_row = tid_in_threadblock_y;
-  const uint32_t local_col = tid_in_threadblock_x;
-  const uint32_t global_row = threadblock_id_y * threadblock_dim_y + local_row;
-  const uint32_t global_col = threadblock_id_x * threadblock_dim_x + local_col;
+  // FIXME: Output block size is assumed to be square, i.e. BM == BN
+  // const uint32_t BM = threadblock_dim_y;
+  // const uint32_t BN = threadblock_dim_y;
+  // const uint32_t BK = threadblock_dim_x;
+  // constexpr uint32_t BM = 8;
+  // constexpr uint32_t BN = 8;
+  // constexpr uint32_t BK = 2;

-  // each thread generates one output element
-  float reg_c = 0.0f;
+  const uint32_t local_a_row = tid_in_threadblock / BK;
+  const uint32_t local_a_col = tid_in_threadblock % BK;
+  const uint32_t local_b_row = tid_in_threadblock / BN;
+  const uint32_t local_b_col = tid_in_threadblock % BN;
+  const uint32_t global_a_row = BM * threadblock_id_y + local_a_row;
+  const uint32_t global_b_col = BN * threadblock_id_x + local_b_col;

-  for (uint32_t k = 0; k < dim_k; k += threadblock_dim_x) {
-    float *local_a = (float *)DEV_SMEM_START_ADDR;
-    size_t local_a_elems = threadblock_dim_x * threadblock_dim_y;
-    float *local_b = (float *)DEV_SMEM_START_ADDR + local_a_elems;
+  const uint32_t local_c_row = tid_in_threadblock / (BN / TN);
+  const uint32_t local_c_col = tid_in_threadblock % (BN / TN);

-    uint32_t offset_global_a = dim_k * global_row + (k + local_col);
-    uint32_t offset_global_b = dim_n * (local_row + k) + global_col;
-    // local_a: threadblock_dim_y rows, threadblock_dim_x cols
-    // local_b: threadblock_dim_x rows, threadblock_dim_y cols
-    // threadblock_dim_x == block_k, threadblock_dim_y == block_m == block_n
-    local_a[threadblock_dim_x * local_row + local_col] = global_a[offset_global_a];
-    local_b[threadblock_dim_y * local_col + local_row] = global_b[offset_global_b];
+  // each thread generates TM output element
+  float reg_c[TM * TN] = { 0.0f };
+  float reg_a[TM] = { 0.0f };
+  float reg_b[TN] = { 0.0f };

-    vx_barrier(0, threadblock_dim_y);
-    vx_fence();
+  volatile float *local_a = sharedmem_per_threadblock;
+  // const size_t local_a_elems = threadblock_dim_x * threadblock_dim_y;
+  const size_t local_a_elems = (BM * BK);
+  volatile float *local_b = sharedmem_per_threadblock + local_a_elems;

-    for (uint32_t local_k = 0; local_k < threadblock_dim_x; local_k++) {
-      reg_c += local_a[threadblock_dim_x * local_row + local_k] *
-               local_b[threadblock_dim_y * local_col + local_k];
+  constexpr uint32_t stride_a = (BM * BN) / BK / (TM * TN);
+  constexpr uint32_t stride_b = (BM * BN) / BN / (TM * TN);
+
+  for (uint32_t k = 0; k < dim_k; k += BK) {
+    // Data move from GMEM to SMEM
+    //
+    // Make sure global offset values for A and B are contiguous between
+    // neighboring threads to ensure GMEM coalescing.
+    for (uint32_t load_offset = 0; load_offset < BM; load_offset += stride_a) {
+      const uint32_t global_a_offset =
+          dim_k * (global_a_row + load_offset) + (k + local_a_col);
+      local_a[BK * (local_a_row + load_offset) + local_a_col] =
+          A[global_a_offset];
+    }
+// #pragma GCC unroll 1
+    for (uint32_t load_offset = 0; load_offset < BK; load_offset += stride_b) {
+      const uint32_t global_b_offset =
+          dim_n * (k + local_b_row + load_offset) + global_b_col;
+      local_b[BN * (local_b_row + load_offset) + local_b_col] =
+          B[global_b_offset];
    }

-    vx_barrier(0, threadblock_dim_y);
-    vx_fence();
+    threadblock_barrier(tid_in_threadblock, threadblock_id_in_cluster,
+                        threadblock_dim_y);
+
+    // Compute single tile*tile matmul
+#pragma GCC unroll 2
+    for (uint32_t local_k = 0; local_k < BK; local_k++) {
+      // First, pump data from SMEM->RF
+#pragma GCC unroll TM
+      for (uint32_t res_idx_m = 0; res_idx_m < TM; res_idx_m++) {
+        reg_a[res_idx_m] =
+            local_a[BK * (TM * local_c_row + res_idx_m) + local_k];
+      }
+#pragma GCC unroll TN
+      for (uint32_t res_idx_n = 0; res_idx_n < TN; res_idx_n++) {
+        reg_b[res_idx_n] =
+            local_b[BN * local_k + (TN * local_c_col + res_idx_n)];
+      }
+
+      // Next, compute multiple result elements (TM*TN) by reusing data in RF
+#pragma GCC unroll TM
+      for (uint32_t res_idx_m = 0; res_idx_m < TM; res_idx_m++) {
+#pragma GCC unroll TN
+        for (uint32_t res_idx_n = 0; res_idx_n < TN; res_idx_n++) {
+          // NOTE use of local_b_row
+          reg_c[TN * res_idx_m + res_idx_n] +=
+              reg_a[res_idx_m] * reg_b[res_idx_n];
+          // reg_c[TN * res_idx_m + res_idx_n] +=
+          //     local_a[BK * (TM * local_c_row + res_idx_m) + local_k] *
+          //     local_b[BN * local_k + (TN * local_c_col + res_idx_n)];
+        }
+      }
+    }
+
+    threadblock_barrier(tid_in_threadblock, threadblock_id_in_cluster,
+                        threadblock_dim_y);
  }

-  global_c[dim_n * global_row + global_col] = reg_c;
+  // Store result data from RF to GMEM
+#pragma GCC unroll TM
+  for (uint32_t res_idx_m = 0; res_idx_m < TM; res_idx_m++) {
+#pragma GCC unroll TN
+    for (uint32_t res_idx_n = 0; res_idx_n < TN; res_idx_n++) {
+      // NOTE use of local_b_row and global_b_col here
+      C[dim_n * (BM * threadblock_id_y + TM * local_c_row + res_idx_m) +
+        (BN * threadblock_id_x + TN * local_c_col + res_idx_n)] =
+          reg_c[TN * res_idx_m + res_idx_n];
+    }
+  }
 }

-void kernel_body(int task_id, kernel_arg_t* __UNIFORM__ arg) {
-  const uint32_t dim_n = arg->dim_n;
-  int tid_x = task_id % dim_n;
-  int tid_y = task_id / dim_n;
+void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
+  // @perf: All threads are running these compute whose result is mostly same
+  // across the threadblock

+  const uint32_t threads_per_threadblock = (BM * BN) / (TM * TN);
+#ifdef RADIANCE
+  const uint32_t threadblocks_per_core = vx_num_threads() * vx_num_warps() /
+                                         threads_per_threadblock *
+                                         CORES_PER_CLUSTER;
+#else
+  const uint32_t threadblocks_per_core =
+      vx_num_threads() * vx_num_warps() / threads_per_threadblock;
+#endif
  const uint32_t threadblock_dim_x = vx_num_threads();
-  const uint32_t threadblock_dim_y = vx_num_warps();
-  const uint32_t threads_per_threadblock = threadblock_dim_x * threadblock_dim_y;
+  const uint32_t threadblock_dim_y = vx_num_warps() / threadblocks_per_core;
  const int threadblock_id = task_id / threads_per_threadblock;
+  const int threadblock_id_in_cluster = threadblock_id % threadblocks_per_core;
+  const int tid_in_threadblock = task_id % threads_per_threadblock;

-  const uint32_t dim_n_in_blocks = dim_n / threadblock_dim_x;
+  const uint32_t dim_m = arg->dim_m;
+  const uint32_t dim_n = arg->dim_n;
+  const uint32_t dim_n_in_blocks = dim_n / BN;
  const int threadblock_id_x = threadblock_id % dim_n_in_blocks;
  const int threadblock_id_y = threadblock_id / dim_n_in_blocks;

-  const int tid_in_threadblock_x = vx_thread_id();
-  const int tid_in_threadblock_y = vx_warp_id() % threadblock_dim_y;
-  thread_block_gemm(arg, tid_in_threadblock_x, tid_in_threadblock_y, threadblock_dim_x,
-                    threadblock_dim_y, threadblock_id_x, threadblock_id_y);
+  // "static" shared memory allocation.  This would determine threadblock
+  // occupancy of a single cluster
+  float *sharedmem_per_threadblock =
+      (float *)DEV_SMEM_START_ADDR + (2 * BM * BK) * threadblock_id_in_cluster;
+  thread_block_gemm(arg, tid_in_threadblock, threadblock_dim_x,
+                    threadblock_dim_y, threadblock_id_x, threadblock_id_y,
+                    threadblock_id_in_cluster, sharedmem_per_threadblock);
 }

 int main() {
  kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
-  const uint32_t grid_size = arg->dim_m * arg->dim_n;
-  vx_spawn_tasks(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
+  const uint32_t grid_size = arg->dim_m * arg->dim_n / (TM * TN);
+#ifdef RADIANCE
+  vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
+#else
+  // NOTE: This kernel assumes contiguous thread scheduling for efficient shared
+  // memory allocation, and therefore does not work with original vx_spawn_tasks
+  vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
+#endif
  return 0;
 }
--- a/tests/regression/sgemm_wg/main.cpp
+++ b/tests/regression/sgemm_wg/main.cpp
@@ -147,8 +147,8 @@ int main(int argc, char *argv[]) {
  RT_CHECK(vx_dev_open(&device));

  // FIXME: hardcoded
-  uint32_t dim_m = 16;
-  uint32_t dim_n = 16;
+  uint32_t dim_m = 32;
+  uint32_t dim_n = 32;
  uint32_t dim_k = 32;

  generate_source_matrix(dim_m, dim_n, dim_k);
--- a/tests/regression/vecaddx/common.h
+++ b/tests/regression/vecaddx/common.h
@@ -1,7 +1,7 @@
 #ifndef _COMMON_H_
 #define _COMMON_H_

-#define KERNEL_ARG_DEV_MEM_ADDR 0x7ffff000
+#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000

 #ifndef TYPE
 #define TYPE float
--- a/tests/regression/vecaddx/kernel.cpp
+++ b/tests/regression/vecaddx/kernel.cpp
@@ -13,6 +13,10 @@ void kernel_body(int task_id, kernel_arg_t* __UNIFORM__ arg) {

 int main() {
 	kernel_arg_t* arg = (kernel_arg_t*)KERNEL_ARG_DEV_MEM_ADDR;
+#ifdef RADIANCE
+	vx_spawn_tasks_cluster(arg->num_points, (vx_spawn_tasks_cb)kernel_body, arg);
+#else
 	vx_spawn_tasks(arg->num_points, (vx_spawn_tasks_cb)kernel_body, arg);
+#endif
 	return 0;
 }
--- a/tests/regression/vecaddx/main.cpp
+++ b/tests/regression/vecaddx/main.cpp
@@ -1,4 +1,5 @@
 #include <iostream>
+#include <fstream>
 #include <unistd.h>
 #include <string.h>
 #include <vector>
@@ -106,9 +107,9 @@ static void parse_args(int argc, char **argv) {

 void cleanup() {
  if (device) {    
-    vx_mem_free(device, kernel_arg.src0_addr);
-    vx_mem_free(device, kernel_arg.src1_addr);
-    vx_mem_free(device, kernel_arg.dst_addr);
+    // vx_mem_free(device, kernel_arg.src0_addr);
+    // vx_mem_free(device, kernel_arg.src1_addr);
+    // vx_mem_free(device, kernel_arg.dst_addr);
    vx_dev_close(device);
  }
 }
@@ -181,9 +182,12 @@ int main(int argc, char *argv[]) {

  // allocate device memory
  std::cout << "allocate device memory" << std::endl;
-  RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.src0_addr));
-  RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.src1_addr));
-  RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.dst_addr));
+  // RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.src0_addr));
+  // RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.src1_addr));
+  // RT_CHECK(vx_mem_alloc(device, buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.dst_addr));
+  kernel_arg.src0_addr = 0x20000UL;
+  kernel_arg.src1_addr = 0x28000UL;
+  kernel_arg.dst_addr = 0xc0000000UL;

  kernel_arg.num_points = num_points;

@@ -201,10 +205,19 @@ int main(int argc, char *argv[]) {
  memcpy(staging_buf.data(), &kernel_arg, sizeof(kernel_arg_t));
  RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t)));

+  std::ofstream file("args.bin", std::ios::binary | std::ios::out);
+  if (!file) {
+    std::cerr << "error: failed to open args.bin for writing\n";
+    exit(EXIT_FAILURE);
+  }
+  file.write(reinterpret_cast<char *>(staging_buf.data()), sizeof(kernel_arg_t));
+  file.close();
+
  // generate source data
  source_data.resize(2 * num_points);
  for (uint32_t i = 0; i < source_data.size(); ++i) {
-    source_data[i] = Comparator<TYPE>::generate();
+    // source_data[i] = Comparator<TYPE>::generate();
+    source_data[i] = static_cast<float>(i);
  }

  // upload source buffer0
@@ -215,6 +228,14 @@ int main(int argc, char *argv[]) {
      buf_ptr[i] = source_data[2 * i + 0];
    }
    RT_CHECK(vx_copy_to_dev(device, kernel_arg.src0_addr, staging_buf.data(), buf_size));
+
+    std::ofstream file("input.a.bin", std::ios::binary | std::ios::out);
+    if (!file) {
+      std::cerr << "error: failed to open input.a.bin for writing\n";
+      exit(EXIT_FAILURE);
+    }
+    file.write(reinterpret_cast<char *>(buf_ptr), buf_size);
+    file.close();
  }

  // upload source buffer1
@@ -225,6 +246,14 @@ int main(int argc, char *argv[]) {
      buf_ptr[i] = source_data[2 * i + 1];
    }   
    RT_CHECK(vx_copy_to_dev(device, kernel_arg.src1_addr, staging_buf.data(), buf_size));
+
+    std::ofstream file("input.b.bin", std::ios::binary | std::ios::out);
+    if (!file) {
+      std::cerr << "error: failed to open input.b.bin for writing\n";
+      exit(EXIT_FAILURE);
+    }
+    file.write(reinterpret_cast<char *>(buf_ptr), buf_size);
+    file.close();
  }

  // clear destination buffer
@@ -243,4 +272,4 @@ int main(int argc, char *argv[]) {
  std::cout << "PASSED!" << std::endl;

  return 0;
-}
+}