diff --git a/hw/rtl/VX_core_wrapper.sv b/hw/rtl/VX_core_wrapper.sv index a8ab65b3..573446d7 100644 --- a/hw/rtl/VX_core_wrapper.sv +++ b/hw/rtl/VX_core_wrapper.sv @@ -104,6 +104,15 @@ module Vortex import VX_gpu_pkg::*; #( output [NUM_TENSOR_CORES*9-1:0] tc_tmem_C_waddr, output [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN-1:0] tc_tmem_C_wdata, output [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN/8-1:0] tc_tmem_C_mask, + output sc_tmem_ren, + input sc_tmem_rready, + output [8:0] sc_tmem_raddr, + input [`NUM_THREADS*`XLEN-1:0] sc_tmem_rdata, + output sc_tmem_wen, + input sc_tmem_wready, + output [8:0] sc_tmem_waddr, + output [`NUM_THREADS*`XLEN-1:0] sc_tmem_wdata, + output [`NUM_THREADS*`XLEN/8-1:0] sc_tmem_mask, // gbar ------------------------------------------------ @@ -514,6 +523,15 @@ module Vortex import VX_gpu_pkg::*; #( .tensor_tmem_C_waddr(tc_tmem_C_waddr), .tensor_tmem_C_wdata(tc_tmem_C_wdata), .tensor_tmem_C_mask(tc_tmem_C_mask), + .scalar_tmem_ren(sc_tmem_ren), + .scalar_tmem_rready(sc_tmem_rready), + .scalar_tmem_raddr(sc_tmem_raddr), + .scalar_tmem_rdata(sc_tmem_rdata), + .scalar_tmem_wen(sc_tmem_wen), + .scalar_tmem_wready(sc_tmem_wready), + .scalar_tmem_waddr(sc_tmem_waddr), + .scalar_tmem_wdata(sc_tmem_wdata), + .scalar_tmem_mask(sc_tmem_mask), .tensor_smem_B_if (tc_p2_bus_if), `else .tensor_tmem_A_ren(tc_tmem_A_ren), @@ -529,6 +547,15 @@ module Vortex import VX_gpu_pkg::*; #( .tensor_tmem_C_waddr(tc_tmem_C_waddr), .tensor_tmem_C_wdata(tc_tmem_C_wdata), .tensor_tmem_C_mask(tc_tmem_C_mask), + .scalar_tmem_ren(sc_tmem_ren), + .scalar_tmem_rready(sc_tmem_rready), + .scalar_tmem_raddr(sc_tmem_raddr), + .scalar_tmem_rdata(sc_tmem_rdata), + .scalar_tmem_wen(sc_tmem_wen), + .scalar_tmem_wready(sc_tmem_wready), + .scalar_tmem_waddr(sc_tmem_waddr), + .scalar_tmem_wdata(sc_tmem_wdata), + .scalar_tmem_mask(sc_tmem_mask), .tensor_smem_B_if (tc_p2_bus_if), `endif diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index 084f0399..b164f945 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -206,9 +206,11 @@ `define INST_LSU_LWU 4'b0110 // new for RV64I LWU `define INST_LSU_SB 4'b1000 `define INST_LSU_SH 4'b1001 -`define INST_LSU_SW 4'b1010 -`define INST_LSU_SD 4'b1011 // new for RV64I SD -`define INST_LSU_FENCE 4'b1111 +`define INST_LSU_SW 4'b1010 +`define INST_LSU_SD 4'b1011 // new for RV64I SD +`define INST_LSU_TMEM_LD 4'b1100 +`define INST_LSU_TMEM_ST 4'b1101 +`define INST_LSU_FENCE 4'b1111 `define INST_LSU_BITS 4 `define INST_LSU_FMT(op) op[2:0] `define INST_LSU_WSIZE(op) op[1:0] diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index a315102c..f2bbafd2 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -54,6 +54,15 @@ module VX_core import VX_gpu_pkg::*; #( output logic [NUM_TENSOR_CORES*9-1:0] tensor_tmem_C_waddr, output logic [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN-1:0] tensor_tmem_C_wdata, output logic [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN/8-1:0] tensor_tmem_C_mask, + output logic scalar_tmem_ren, + input logic scalar_tmem_rready, + output logic [8:0] scalar_tmem_raddr, + input logic [`NUM_THREADS*`XLEN-1:0] scalar_tmem_rdata, + output logic scalar_tmem_wen, + input logic scalar_tmem_wready, + output logic [8:0] scalar_tmem_waddr, + output logic [`NUM_THREADS*`XLEN-1:0] scalar_tmem_wdata, + output logic [`NUM_THREADS*`XLEN/8-1:0] scalar_tmem_mask, VX_tc_bus_if.master tensor_smem_B_if[NUM_TENSOR_CORES], `ifdef GBAR_ENABLE @@ -410,7 +419,16 @@ module VX_core import VX_gpu_pkg::*; #( .tensor_tmem_C_waddr(tensor_tmem_C_waddr), .tensor_tmem_C_wdata(tensor_tmem_C_wdata), .tensor_tmem_C_mask(tensor_tmem_C_mask), - .tensor_smem_B_if (tensor_smem_B_if), + .scalar_tmem_ren(scalar_tmem_ren), + .scalar_tmem_rready(scalar_tmem_rready), + .scalar_tmem_raddr(scalar_tmem_raddr), + .scalar_tmem_rdata(scalar_tmem_rdata), + .scalar_tmem_wen(scalar_tmem_wen), + .scalar_tmem_wready(scalar_tmem_wready), + .scalar_tmem_waddr(scalar_tmem_waddr), + .scalar_tmem_wdata(scalar_tmem_wdata), + .scalar_tmem_mask(scalar_tmem_mask), + .tensor_smem_B_if (tensor_smem_B_if), `endif `endif diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 44d06da0..ed81c5cf 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -508,9 +508,24 @@ module VX_decode #( end `INST_EXT2: begin case (func3) + 3'h0: begin + if (func7 == 7'h30) begin + ex_type = `EX_LSU; + op_type = `INST_LSU_TMEM_LD; + use_rd = 1; + `USED_IREG (rd); + `USED_IREG (rs1); + end + end 3'h1: begin - case (func2) - 2'h0: begin // CMOV + if (func7 == 7'h30) begin + ex_type = `EX_LSU; + op_type = `INST_LSU_TMEM_ST; + `USED_IREG (rs1); + `USED_IREG (rs2); + end else begin + case (func2) + 2'h0: begin // CMOV ex_type = `EX_SFU; op_type = `INST_OP_BITS'(`INST_SFU_CMOV); use_rd = 1; @@ -518,9 +533,10 @@ module VX_decode #( `USED_IREG (rs1); `USED_IREG (rs2); `USED_IREG (rs3); - end - default:; - endcase + end + default:; + endcase + end end default:; endcase diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index b169933b..7a4e3d65 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -92,6 +92,16 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `endif + output logic scalar_tmem_ren, + input logic scalar_tmem_rready, + output logic [8:0] scalar_tmem_raddr, + input logic [`NUM_THREADS*`XLEN-1:0] scalar_tmem_rdata, + output logic scalar_tmem_wen, + input logic scalar_tmem_wready, + output logic [8:0] scalar_tmem_waddr, + output logic [`NUM_THREADS*`XLEN-1:0] scalar_tmem_wdata, + output logic [`NUM_THREADS*`XLEN/8-1:0] scalar_tmem_mask, + // simulation helper signals output wire sim_ebreak, @@ -286,8 +296,34 @@ module VX_execute import VX_gpu_pkg::*; #( `SCOPE_IO_SWITCH (1) + VX_dispatch_if scalar_mem_lsu_dispatch_if[`ISSUE_WIDTH](); + VX_commit_if scalar_mem_lsu_commit_if[`ISSUE_WIDTH](); + VX_commit_if scalar_tmem_commit_if[`ISSUE_WIDTH](); VX_commit_if lsu_scalar_commit_if[`ISSUE_WIDTH](); + wire [`ISSUE_WIDTH-1:0] scalar_tmem_dispatch; + wire [`ISSUE_WIDTH-1:0] scalar_tmem_ld_dispatch; + wire [`ISSUE_WIDTH-1:0] scalar_tmem_st_dispatch; + wire [`ISSUE_WIDTH-1:0] scalar_tmem_dispatch_ready; + wire [`ISSUE_WIDTH-1:0] scalar_tmem_dispatch_fire; + wire [`ISSUE_WIDTH-1:0] scalar_tmem_commit_ready; + `UNUSED_VAR (scalar_tmem_dispatch_fire) + + for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin : g_scalar_lsu_split + assign scalar_tmem_ld_dispatch[i] = lsu_dispatch_if[i].valid + && (lsu_dispatch_if[i].data.op_type == `INST_LSU_TMEM_LD); + assign scalar_tmem_st_dispatch[i] = lsu_dispatch_if[i].valid + && (lsu_dispatch_if[i].data.op_type == `INST_LSU_TMEM_ST); + assign scalar_tmem_dispatch[i] = scalar_tmem_ld_dispatch[i] || scalar_tmem_st_dispatch[i]; + + assign scalar_mem_lsu_dispatch_if[i].valid = lsu_dispatch_if[i].valid && !scalar_tmem_dispatch[i]; + assign scalar_mem_lsu_dispatch_if[i].data = lsu_dispatch_if[i].data; + assign lsu_dispatch_if[i].ready = scalar_tmem_dispatch[i] ? scalar_tmem_dispatch_ready[i] + : scalar_mem_lsu_dispatch_if[i].ready; + + assign scalar_tmem_dispatch_fire[i] = scalar_tmem_dispatch[i] && scalar_tmem_dispatch_ready[i]; + end + VX_mem_bus_if #( .DATA_SIZE (DCACHE_WORD_SIZE), .TAG_WIDTH (DCACHE_TAG_WIDTH) @@ -301,10 +337,169 @@ module VX_execute import VX_gpu_pkg::*; #( .reset (lsu_reset), .downstream_mem_busy (downstream_mem_busy), .cache_bus_if (scalar_lsu_bus_if), - .dispatch_if (lsu_dispatch_if), - .commit_if (lsu_scalar_commit_if) + .dispatch_if (scalar_mem_lsu_dispatch_if), + .commit_if (scalar_mem_lsu_commit_if) ); + wire scalar_tmem_pending; + reg [`ISSUE_WIDTH-1:0] scalar_tmem_grant; + reg [`ISSUE_WIDTH-1:0] scalar_tmem_grant_r; + reg scalar_tmem_load_pending; + reg scalar_tmem_store_pending; + reg scalar_tmem_commit_pending; + reg [`UUID_WIDTH-1:0] scalar_tmem_uuid_r; + reg [`NW_WIDTH-1:0] scalar_tmem_wid_r; + reg [`NUM_THREADS-1:0] scalar_tmem_tmask_r; + reg [`XLEN-1:0] scalar_tmem_pc_r; + reg [`NR_BITS-1:0] scalar_tmem_rd_r; + reg [`NUM_THREADS*`XLEN-1:0] scalar_tmem_rdata_r; + reg scalar_tmem_rdata_valid_r; + wire [`ISSUE_WIDTH-1:0][8:0] scalar_tmem_req_addr; + wire [`ISSUE_WIDTH-1:0][`NUM_THREADS*`XLEN-1:0] scalar_tmem_req_wdata; + wire [`ISSUE_WIDTH-1:0][`UUID_WIDTH-1:0] scalar_tmem_req_uuid; + wire [`ISSUE_WIDTH-1:0][`NW_WIDTH-1:0] scalar_tmem_req_wid; + wire [`ISSUE_WIDTH-1:0][`NUM_THREADS-1:0] scalar_tmem_req_tmask; + wire [`ISSUE_WIDTH-1:0][`XLEN-1:0] scalar_tmem_req_pc; + wire [`ISSUE_WIDTH-1:0][`NR_BITS-1:0] scalar_tmem_req_rd; + + assign scalar_tmem_pending = |scalar_tmem_dispatch; + + always @(*) begin + scalar_tmem_grant = '0; + for (integer i = `ISSUE_WIDTH-1; i >= 0; --i) begin + if (scalar_tmem_dispatch[i]) begin + scalar_tmem_grant = '0; + scalar_tmem_grant[i] = 1'b1; + end + end + end + + wire scalar_tmem_grant_valid = scalar_tmem_pending && !scalar_tmem_load_pending + && !scalar_tmem_store_pending && !scalar_tmem_commit_pending; + wire scalar_tmem_grant_is_load = |(scalar_tmem_grant & scalar_tmem_ld_dispatch); + wire scalar_tmem_grant_is_store = |(scalar_tmem_grant & scalar_tmem_st_dispatch); + wire scalar_tmem_req_ready = scalar_tmem_grant_is_load ? scalar_tmem_rready : scalar_tmem_wready; + wire scalar_tmem_req_fire = scalar_tmem_grant_valid && scalar_tmem_req_ready; + + for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin : g_scalar_tmem_ready + assign scalar_tmem_dispatch_ready[i] = scalar_tmem_grant_valid && scalar_tmem_grant[i] && scalar_tmem_req_ready; + assign scalar_tmem_req_addr[i] = lsu_dispatch_if[i].data.rs1_data[0][8:0]; + assign scalar_tmem_req_wdata[i] = lsu_dispatch_if[i].data.rs2_data; + assign scalar_tmem_req_uuid[i] = lsu_dispatch_if[i].data.uuid; + assign scalar_tmem_req_wid[i] = wis_to_wid(lsu_dispatch_if[i].data.wis, ISSUE_ISW_W'(i)); + assign scalar_tmem_req_tmask[i] = lsu_dispatch_if[i].data.tmask; + assign scalar_tmem_req_pc[i] = lsu_dispatch_if[i].data.PC; + assign scalar_tmem_req_rd[i] = lsu_dispatch_if[i].data.rd; + + assign scalar_tmem_commit_if[i].valid = scalar_tmem_commit_pending && scalar_tmem_grant_r[i]; + assign scalar_tmem_commit_if[i].data.uuid = scalar_tmem_uuid_r; + assign scalar_tmem_commit_if[i].data.wid = scalar_tmem_wid_r; + assign scalar_tmem_commit_if[i].data.tmask = scalar_tmem_tmask_r; + assign scalar_tmem_commit_if[i].data.PC = scalar_tmem_pc_r; + assign scalar_tmem_commit_if[i].data.wb = scalar_tmem_load_pending; + assign scalar_tmem_commit_if[i].data.rd = scalar_tmem_load_pending ? scalar_tmem_rd_r : '0; + assign scalar_tmem_commit_if[i].data.data = scalar_tmem_rdata_valid_r ? scalar_tmem_rdata_r : scalar_tmem_rdata; + assign scalar_tmem_commit_if[i].data.tensor = 1'b0; + assign scalar_tmem_commit_if[i].data.pid = '0; + assign scalar_tmem_commit_if[i].data.sop = 1'b1; + assign scalar_tmem_commit_if[i].data.eop = 1'b1; + assign scalar_tmem_commit_ready[i] = scalar_tmem_commit_if[i].ready; + end + + function automatic [`NUM_THREADS*`XLEN/8-1:0] scalar_tmem_expand_tmask; + input [`NUM_THREADS-1:0] tmask; + begin + scalar_tmem_expand_tmask = '0; + for (integer lane = 0; lane < `NUM_THREADS; ++lane) begin + scalar_tmem_expand_tmask[lane * (`XLEN / 8) +: (`XLEN / 8)] = + {(`XLEN / 8){tmask[lane]}}; + end + end + endfunction + + always @(*) begin + scalar_tmem_ren = scalar_tmem_grant_valid && scalar_tmem_grant_is_load; + scalar_tmem_wen = scalar_tmem_grant_valid && scalar_tmem_grant_is_store; + scalar_tmem_raddr = '0; + scalar_tmem_waddr = '0; + scalar_tmem_wdata = '0; + scalar_tmem_mask = '0; + for (integer i = 0; i < `ISSUE_WIDTH; ++i) begin + if (scalar_tmem_grant[i]) begin + scalar_tmem_raddr = scalar_tmem_req_addr[i]; + scalar_tmem_waddr = scalar_tmem_req_addr[i]; + scalar_tmem_wdata = scalar_tmem_req_wdata[i]; + scalar_tmem_mask = scalar_tmem_expand_tmask(scalar_tmem_req_tmask[i]) + & {(`NUM_THREADS * (`XLEN / 8)){scalar_tmem_grant_is_store}}; + end + end + end + + always @(posedge clk) begin + if (reset) begin + scalar_tmem_grant_r <= '0; + scalar_tmem_load_pending <= 1'b0; + scalar_tmem_store_pending <= 1'b0; + scalar_tmem_commit_pending <= 1'b0; + scalar_tmem_uuid_r <= '0; + scalar_tmem_wid_r <= '0; + scalar_tmem_tmask_r <= '0; + scalar_tmem_pc_r <= '0; + scalar_tmem_rd_r <= '0; + scalar_tmem_rdata_r <= '0; + scalar_tmem_rdata_valid_r <= 1'b0; + end else begin + if (scalar_tmem_req_fire) begin + scalar_tmem_grant_r <= scalar_tmem_grant; + scalar_tmem_load_pending <= scalar_tmem_grant_is_load; + scalar_tmem_store_pending <= scalar_tmem_grant_is_store; + scalar_tmem_commit_pending <= 1'b1; + scalar_tmem_rdata_valid_r <= 1'b0; + for (integer i = 0; i < `ISSUE_WIDTH; ++i) begin + if (scalar_tmem_grant[i]) begin + scalar_tmem_uuid_r <= scalar_tmem_req_uuid[i]; + scalar_tmem_wid_r <= scalar_tmem_req_wid[i]; + scalar_tmem_tmask_r <= scalar_tmem_req_tmask[i]; + scalar_tmem_pc_r <= scalar_tmem_req_pc[i]; + scalar_tmem_rd_r <= scalar_tmem_req_rd[i]; + end + end + end else if (scalar_tmem_load_pending && scalar_tmem_commit_pending && !scalar_tmem_rdata_valid_r) begin + scalar_tmem_rdata_r <= scalar_tmem_rdata; + scalar_tmem_rdata_valid_r <= 1'b1; + end + + if (scalar_tmem_commit_pending && (|(scalar_tmem_grant_r & scalar_tmem_commit_ready))) begin + scalar_tmem_grant_r <= '0; + scalar_tmem_load_pending <= 1'b0; + scalar_tmem_store_pending <= 1'b0; + scalar_tmem_commit_pending <= 1'b0; + scalar_tmem_rdata_valid_r <= 1'b0; + end + end + end + + localparam SCALAR_LSU_COMMIT_DATAW = `UUID_WIDTH + `NW_WIDTH + `NUM_THREADS + `XLEN + 1 + `NR_BITS + (`NUM_THREADS * `XLEN) + 1 + 1 + 1 + 1; + + for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin : g_scalar_lsu_commit + VX_stream_arb #( + .NUM_INPUTS (2), + .DATAW (SCALAR_LSU_COMMIT_DATAW), + .ARBITER ("R"), + .OUT_REG (1) + ) scalar_lsu_commit_arb ( + .clk (clk), + .reset (reset), + .valid_in ({scalar_tmem_commit_if[i].valid, scalar_mem_lsu_commit_if[i].valid}), + .ready_in ({scalar_tmem_commit_if[i].ready, scalar_mem_lsu_commit_if[i].ready}), + .data_in ({scalar_tmem_commit_if[i].data, scalar_mem_lsu_commit_if[i].data}), + .data_out (lsu_scalar_commit_if[i].data), + .valid_out (lsu_scalar_commit_if[i].valid), + .ready_out (lsu_scalar_commit_if[i].ready), + `UNUSED_PIN (sel_out) + ); + end + `ifdef EXT_T_ENABLE VX_commit_if lsu_tensor_commit_if[`ISSUE_WIDTH]();