fix: pack scalar tmem softmax as fp16

hw/dpi/float_dpi.cpp

@@ -12,6 +12,7 @@
 // limitations under the License.
 
 #include <stdio.h>
+#include <stdint.h>
 #include <math.h>
 #include <unordered_map>
 #include <vector>
@@ -37,6 +38,11 @@ extern "C" {
 
   void dpi_fdiv(bool enable, int dst_fmt, int64_t a, int64_t b, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags);
   void dpi_fsqrt(bool enable, int dst_fmt, int64_t a, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags);
+  void dpi_fexp(bool enable, int dst_fmt, int64_t a, int64_t* result, svBitVecVal* fflags);
+  int dpi_f32_max(bool enable, int a_bits, int b_bits);
+  int dpi_softmax_exp_acc(bool enable, int score_bits, int max_bits, int accum_bits);
+  int dpi_softmax_prob_to_f16x2(bool enable, int score_bits, int max_bits, int denom_bits);
+  int dpi_softmax_prob_to_fp8e4m3x4(bool enable, int score_bits, int max_bits, int denom_bits);
 
   void dpi_ftoi(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags);
   void dpi_ftou(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags);
@@ -83,6 +89,89 @@ inline int64_t check_boxing(int64_t a) {
   return a;
 }
 
+inline float bits_to_float(uint32_t value) {
+  union {
+    uint32_t u;
+    float f;
+  } bits = {value};
+  return bits.f;
+}
+
+inline uint32_t float_to_bits(float value) {
+  union {
+    float f;
+    uint32_t u;
+  } bits = {value};
+  return bits.u;
+}
+
+int dpi_f32_max(bool enable, int a_bits, int b_bits) {
+  if (!enable)
+    return 0;
+  float a = bits_to_float(static_cast<uint32_t>(a_bits));
+  float b = bits_to_float(static_cast<uint32_t>(b_bits));
+  return static_cast<int>(float_to_bits(a > b ? a : b));
+}
+
+int dpi_softmax_exp_acc(bool enable, int score_bits, int max_bits, int accum_bits) {
+  if (!enable)
+    return 0;
+  float score = bits_to_float(static_cast<uint32_t>(score_bits));
+  float row_max = bits_to_float(static_cast<uint32_t>(max_bits));
+  float accum = bits_to_float(static_cast<uint32_t>(accum_bits));
+  return static_cast<int>(float_to_bits(accum + expf(score - row_max)));
+}
+
+int dpi_softmax_prob_to_f16x2(bool enable, int score_bits, int max_bits, int denom_bits) {
+  if (!enable)
+    return 0;
+  float score = bits_to_float(static_cast<uint32_t>(score_bits));
+  float row_max = bits_to_float(static_cast<uint32_t>(max_bits));
+  float denom = bits_to_float(static_cast<uint32_t>(denom_bits));
+  float prob = denom == 0.0f ? 0.0f : expf(score - row_max) / denom;
+  half_float::half half_prob(prob);
+  uint16_t half_bits;
+  static_assert(sizeof(half_bits) == sizeof(half_prob), "unexpected half size");
+  __builtin_memcpy(&half_bits, &half_prob, sizeof(half_bits));
+  return static_cast<int>(half_bits | (uint32_t(half_bits) << 16));
+}
+
+static float fp8e4m3_positive_to_float(uint8_t bits) {
+  const int exp = (bits >> 3) & 0xf;
+  const int frac = bits & 0x7;
+  if (exp == 0) {
+    return frac == 0 ? 0.0f : ldexpf(static_cast<float>(frac) / 8.0f, -6);
+  }
+  return ldexpf(1.0f + static_cast<float>(frac) / 8.0f, exp - 7);
+}
+
+static uint8_t float_to_fp8e4m3_positive(float value) {
+  if (!(value > 0.0f))
+    return 0;
+  uint8_t best = 0;
+  float best_diff = fabsf(value);
+  for (uint32_t bits = 1; bits < 0x80; ++bits) {
+    const float candidate = fp8e4m3_positive_to_float(static_cast<uint8_t>(bits));
+    const float diff = fabsf(candidate - value);
+    if (diff < best_diff || (diff == best_diff && ((bits & 1u) == 0u))) {
+      best = static_cast<uint8_t>(bits);
+      best_diff = diff;
+    }
+  }
+  return best;
+}
+
+int dpi_softmax_prob_to_fp8e4m3x4(bool enable, int score_bits, int max_bits, int denom_bits) {
+  if (!enable)
+    return 0;
+  float score = bits_to_float(static_cast<uint32_t>(score_bits));
+  float row_max = bits_to_float(static_cast<uint32_t>(max_bits));
+  float denom = bits_to_float(static_cast<uint32_t>(denom_bits));
+  float prob = denom == 0.0f ? 0.0f : expf(score - row_max) / denom;
+  uint32_t fp8 = float_to_fp8e4m3_positive(prob);
+  return static_cast<int>(fp8 | (fp8 << 8) | (fp8 << 16) | (fp8 << 24));
+}
+
 void dpi_fadd(bool enable, int dst_fmt, int64_t a, int64_t b, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags) {
   if (!enable) 
     return;
@@ -173,6 +262,38 @@ void dpi_fsqrt(bool enable, int dst_fmt, int64_t a, const svBitVecVal* frm, int6
   }
 }
 
+void dpi_fexp(bool enable, int dst_fmt, int64_t a, int64_t* result, svBitVecVal* fflags) {
+  if (!enable)
+    return;
+  *fflags = 0;
+  if (dst_fmt) {
+    double input;
+    static_assert(sizeof(input) == sizeof(a), "unexpected double size");
+    __builtin_memcpy(&input, &a, sizeof(input));
+    double output = exp(input);
+    __builtin_memcpy(result, &output, sizeof(output));
+    if (isinf(output) && isfinite(input)) {
+      *fflags |= 0x05; // OF | NX
+    } else if (output == 0.0 && input < 0.0 && isfinite(input)) {
+      *fflags |= 0x03; // UF | NX
+    } else if (isfinite(input)) {
+      *fflags |= 0x01; // NX
+    }
+  } else {
+    uint32_t boxed = static_cast<uint32_t>(check_boxing(a));
+    float input = bits_to_float(boxed);
+    float output = expf(input);
+    *result = nan_box(float_to_bits(output));
+    if (isinf(output) && isfinite(input)) {
+      *fflags |= 0x05; // OF | NX
+    } else if (output == 0.0f && input < 0.0f && isfinite(input)) {
+      *fflags |= 0x03; // UF | NX
+    } else if (isfinite(input)) {
+      *fflags |= 0x01; // NX
+    }
+  }
+}
+
 void dpi_ftoi(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags) {
   if (!enable) 
     return;

hw/dpi/float_dpi.vh

@@ -26,6 +26,11 @@ import "DPI-C" function void dpi_fnmsub(input logic enable, input int dst_fmt, i
 
 import "DPI-C" function void dpi_fdiv(input logic enable, input int dst_fmt, input longint a, input longint b, input bit[2:0] frm, output longint result, output bit[4:0] fflags);
 import "DPI-C" function void dpi_fsqrt(input logic enable, input int dst_fmt, input longint a, input bit[2:0] frm, output longint result, output bit[4:0] fflags);
+import "DPI-C" function void dpi_fexp(input logic enable, input int dst_fmt, input longint a, output longint result, output bit[4:0] fflags);
+import "DPI-C" function int dpi_f32_max(input logic enable, input int a_bits, input int b_bits);
+import "DPI-C" function int dpi_softmax_exp_acc(input logic enable, input int score_bits, input int max_bits, input int accum_bits);
+import "DPI-C" function int dpi_softmax_prob_to_f16x2(input logic enable, input int score_bits, input int max_bits, input int denom_bits);
+import "DPI-C" function int dpi_softmax_prob_to_fp8e4m3x4(input logic enable, input int score_bits, input int max_bits, input int denom_bits);
 
 import "DPI-C" function void dpi_ftoi(input logic enable, input int dst_fmt, input int src_fmt, input longint a, input bit[2:0] frm, output longint result, output bit[4:0] fflags);
 import "DPI-C" function void dpi_ftou(input logic enable, input int dst_fmt, input int src_fmt, input longint a, input bit[2:0] frm, output longint result, output bit[4:0] fflags);

hw/rtl/VX_config.vh

@@ -422,6 +422,11 @@
 `endif
 `endif
 
+// FEXP Latency
+`ifndef LATENCY_FEXP
+`define LATENCY_FEXP 8
+`endif
+
 // FCVT Latency
 `ifndef LATENCY_FCVT
 `define LATENCY_FCVT 5

hw/rtl/VX_core_wrapper.sv

@@ -104,6 +104,15 @@ module Vortex import VX_gpu_pkg::*; #(
     output [NUM_TENSOR_CORES*9-1:0]                       tc_tmem_C_waddr,
     output [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN-1:0]      tc_tmem_C_wdata,
     output [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN/8-1:0]    tc_tmem_C_mask,
+    output                                                sc_tmem_ren,
+    input                                                 sc_tmem_rready,
+    output [8:0]                                          sc_tmem_raddr,
+    input  [`NUM_THREADS*`XLEN-1:0]                       sc_tmem_rdata,
+    output                                                sc_tmem_wen,
+    input                                                 sc_tmem_wready,
+    output [8:0]                                          sc_tmem_waddr,
+    output [`NUM_THREADS*`XLEN-1:0]                       sc_tmem_wdata,
+    output [`NUM_THREADS*`XLEN/8-1:0]                     sc_tmem_mask,
 
     // gbar ------------------------------------------------
 
@@ -514,6 +523,15 @@ module Vortex import VX_gpu_pkg::*; #(
         .tensor_tmem_C_waddr(tc_tmem_C_waddr),
         .tensor_tmem_C_wdata(tc_tmem_C_wdata),
         .tensor_tmem_C_mask(tc_tmem_C_mask),
+        .scalar_tmem_ren(sc_tmem_ren),
+        .scalar_tmem_rready(sc_tmem_rready),
+        .scalar_tmem_raddr(sc_tmem_raddr),
+        .scalar_tmem_rdata(sc_tmem_rdata),
+        .scalar_tmem_wen(sc_tmem_wen),
+        .scalar_tmem_wready(sc_tmem_wready),
+        .scalar_tmem_waddr(sc_tmem_waddr),
+        .scalar_tmem_wdata(sc_tmem_wdata),
+        .scalar_tmem_mask(sc_tmem_mask),
         .tensor_smem_B_if (tc_p2_bus_if),
     `else
         .tensor_tmem_A_ren(tc_tmem_A_ren),
@@ -529,6 +547,15 @@ module Vortex import VX_gpu_pkg::*; #(
         .tensor_tmem_C_waddr(tc_tmem_C_waddr),
         .tensor_tmem_C_wdata(tc_tmem_C_wdata),
         .tensor_tmem_C_mask(tc_tmem_C_mask),
+        .scalar_tmem_ren(sc_tmem_ren),
+        .scalar_tmem_rready(sc_tmem_rready),
+        .scalar_tmem_raddr(sc_tmem_raddr),
+        .scalar_tmem_rdata(sc_tmem_rdata),
+        .scalar_tmem_wen(sc_tmem_wen),
+        .scalar_tmem_wready(sc_tmem_wready),
+        .scalar_tmem_waddr(sc_tmem_waddr),
+        .scalar_tmem_wdata(sc_tmem_wdata),
+        .scalar_tmem_mask(sc_tmem_mask),
         .tensor_smem_B_if (tc_p2_bus_if),
     `endif
 

hw/rtl/VX_define.vh

@@ -206,9 +206,12 @@
 `define INST_LSU_LWU         4'b0110 // new for RV64I LWU
 `define INST_LSU_SB          4'b1000 
 `define INST_LSU_SH          4'b1001
-`define INST_LSU_SW          4'b1010
-`define INST_LSU_SD          4'b1011 // new for RV64I SD
-`define INST_LSU_FENCE       4'b1111
+`define INST_LSU_SW          4'b1010
+`define INST_LSU_SD          4'b1011 // new for RV64I SD
+`define INST_LSU_TMEM_LD     4'b1100
+`define INST_LSU_TMEM_ST     4'b1101
+`define INST_LSU_TMEM_SOFTMAX 4'b1110
+`define INST_LSU_FENCE       4'b1111
 `define INST_LSU_BITS        4
 `define INST_LSU_FMT(op)     op[2:0]
 `define INST_LSU_WSIZE(op)   op[1:0]
@@ -225,7 +228,7 @@
 `define INST_FPU_SQRT        4'b0100
 `define INST_FPU_CMP         4'b0101 // mod: LE=0, LT=1, EQ=2
 `define INST_FPU_F2F         4'b0110
-`define INST_FPU_MISC        4'b0111 // mod: SGNJ=0, SGNJN=1, SGNJX=2, CLASS=3, MVXW=4, MVWX=5, FMIN=6, FMAX=7
+`define INST_FPU_MISC        4'b0111 // mod: SGNJ=0, SGNJN=1, SGNJX=2, CLASS=3, MVXW=4, MVWX=5, FMIN=6, FMAX=7
 `define INST_FPU_F2I         4'b1000
 `define INST_FPU_F2U         4'b1001
 `define INST_FPU_I2F         4'b1010
@@ -234,10 +237,11 @@
 `define INST_FPU_MSUB        4'b1101   
 `define INST_FPU_NMSUB       4'b1110   
 `define INST_FPU_NMADD       4'b1111
-`define INST_FPU_BITS        4
-`define INST_FPU_IS_W(mod)   (mod[4])
-`define INST_FPU_IS_CLASS(op, mod) (op == `INST_FPU_MISC && mod == 3)
-`define INST_FPU_IS_MVXW(op, mod) (op == `INST_FPU_MISC && mod == 4)
+`define INST_FPU_BITS        4
+`define INST_FPU_IS_W(mod)   (mod[4])
+`define INST_FPU_IS_CLASS(op, mod) (op == `INST_FPU_MISC && mod == 3)
+`define INST_FPU_IS_MVXW(op, mod) (op == `INST_FPU_MISC && mod == 4)
+`define INST_FPU_IS_EXP(op, mod) (op == `INST_FPU_MISC && mod == 8)
 
 `define INST_SFU_TMC         4'h0
 `define INST_SFU_WSPAWN      4'h1 

hw/rtl/core/VX_core.sv

@@ -54,6 +54,15 @@ module VX_core import VX_gpu_pkg::*; #(
     output logic [NUM_TENSOR_CORES*9-1:0]                       tensor_tmem_C_waddr,
     output logic [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN-1:0]      tensor_tmem_C_wdata,
     output logic [NUM_TENSOR_CORES*`NUM_THREADS*`XLEN/8-1:0]    tensor_tmem_C_mask,
+    output logic                                                scalar_tmem_ren,
+    input  logic                                                scalar_tmem_rready,
+    output logic [8:0]                                          scalar_tmem_raddr,
+    input  logic [`NUM_THREADS*`XLEN-1:0]                       scalar_tmem_rdata,
+    output logic                                                scalar_tmem_wen,
+    input  logic                                                scalar_tmem_wready,
+    output logic [8:0]                                          scalar_tmem_waddr,
+    output logic [`NUM_THREADS*`XLEN-1:0]                       scalar_tmem_wdata,
+    output logic [`NUM_THREADS*`XLEN/8-1:0]                     scalar_tmem_mask,
     VX_tc_bus_if.master     tensor_smem_B_if[NUM_TENSOR_CORES],
 
 `ifdef GBAR_ENABLE
@@ -410,7 +419,16 @@ module VX_core import VX_gpu_pkg::*; #(
         .tensor_tmem_C_waddr(tensor_tmem_C_waddr),
         .tensor_tmem_C_wdata(tensor_tmem_C_wdata),
         .tensor_tmem_C_mask(tensor_tmem_C_mask),
-        .tensor_smem_B_if  (tensor_smem_B_if),
+        .scalar_tmem_ren(scalar_tmem_ren),
+        .scalar_tmem_rready(scalar_tmem_rready),
+        .scalar_tmem_raddr(scalar_tmem_raddr),
+        .scalar_tmem_rdata(scalar_tmem_rdata),
+        .scalar_tmem_wen(scalar_tmem_wen),
+        .scalar_tmem_wready(scalar_tmem_wready),
+        .scalar_tmem_waddr(scalar_tmem_waddr),
+        .scalar_tmem_wdata(scalar_tmem_wdata),
+        .scalar_tmem_mask(scalar_tmem_mask),
+        .tensor_smem_B_if  (tensor_smem_B_if),
     `endif
     `endif
 

hw/rtl/core/VX_decode.sv

@@ -508,9 +508,24 @@ module VX_decode  #(
             end
             `INST_EXT2: begin                
                 case (func3)
+                    3'h0: begin
+                        if (func7 == 7'h30) begin
+                            ex_type = `EX_LSU;
+                            op_type = `INST_LSU_TMEM_LD;
+                            use_rd = 1;
+                            `USED_IREG (rd);
+                            `USED_IREG (rs1);
+                        end
+                    end
                     3'h1: begin
-                        case (func2)                       
-                            2'h0: begin // CMOV
+                        if (func7 == 7'h30) begin
+                            ex_type = `EX_LSU;
+                            op_type = `INST_LSU_TMEM_ST;
+                            `USED_IREG (rs1);
+                            `USED_IREG (rs2);
+                        end else begin
+                            case (func2)
+                                2'h0: begin // CMOV
                                 ex_type = `EX_SFU;
                                 op_type = `INST_OP_BITS'(`INST_SFU_CMOV);
                                 use_rd = 1;
@@ -518,9 +533,32 @@ module VX_decode  #(
                                 `USED_IREG (rs1);
                                 `USED_IREG (rs2);
                                 `USED_IREG (rs3);
-                            end
-                            default:;
-                        endcase
+                                end
+                                default:;
+                            endcase
+                        end
+                    end
+                `ifdef EXT_F_ENABLE
+                    3'h2: begin
+                        if (func7 == 7'h30 && rs2 == 5'd0) begin
+                            ex_type = `EX_FPU;
+                            op_type = `INST_OP_BITS'(`INST_FPU_MISC);
+                            op_mod  = `INST_MOD_BITS'(8); // FEXP.S
+                            use_rd  = 1;
+                            `USED_FREG (rd);
+                            `USED_FREG (rs1);
+                        end
+                    end
+                `endif
+                    3'h3: begin
+                        if (func7 == 7'h30) begin
+                            ex_type = `EX_LSU;
+                            op_type = `INST_LSU_TMEM_SOFTMAX;
+                            use_rd = 1;
+                            `USED_IREG (rd);
+                            `USED_IREG (rs1);
+                            `USED_IREG (rs2);
+                        end
                     end
                     default:;
                 endcase

hw/rtl/core/VX_execute.sv

@@ -12,6 +12,9 @@
 // limitations under the License.
 
 `include "VX_define.vh"
+`ifdef SV_DPI
+`include "float_dpi.vh"
+`endif
 
 module VX_execute import VX_gpu_pkg::*; #(
     parameter CORE_ID = 0,
@@ -92,6 +95,16 @@ module VX_execute import VX_gpu_pkg::*; #(
 `endif
 `endif
 
+    output logic                                                scalar_tmem_ren,
+    input  logic                                                scalar_tmem_rready,
+    output logic [8:0]                                          scalar_tmem_raddr,
+    input  logic [`NUM_THREADS*`XLEN-1:0]                       scalar_tmem_rdata,
+    output logic                                                scalar_tmem_wen,
+    input  logic                                                scalar_tmem_wready,
+    output logic [8:0]                                          scalar_tmem_waddr,
+    output logic [`NUM_THREADS*`XLEN-1:0]                       scalar_tmem_wdata,
+    output logic [`NUM_THREADS*`XLEN/8-1:0]                     scalar_tmem_mask,
+
     // simulation helper signals
     output wire             sim_ebreak,
 
@@ -286,8 +299,39 @@ module VX_execute import VX_gpu_pkg::*; #(
 
     `SCOPE_IO_SWITCH (1)
 
+    VX_dispatch_if scalar_mem_lsu_dispatch_if[`ISSUE_WIDTH]();
+    VX_commit_if scalar_mem_lsu_commit_if[`ISSUE_WIDTH]();
+    VX_commit_if scalar_tmem_commit_if[`ISSUE_WIDTH]();
     VX_commit_if lsu_scalar_commit_if[`ISSUE_WIDTH]();
 
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_dispatch;
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_ld_dispatch;
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_st_dispatch;
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_softmax_dispatch;
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_dispatch_ready;
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_dispatch_fire;
+    wire [`ISSUE_WIDTH-1:0] scalar_tmem_commit_ready;
+    `UNUSED_VAR (scalar_tmem_dispatch_fire)
+
+    for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin : g_scalar_lsu_split
+        assign scalar_tmem_ld_dispatch[i] = lsu_dispatch_if[i].valid
+                                         && (lsu_dispatch_if[i].data.op_type == `INST_LSU_TMEM_LD);
+        assign scalar_tmem_st_dispatch[i] = lsu_dispatch_if[i].valid
+                                         && (lsu_dispatch_if[i].data.op_type == `INST_LSU_TMEM_ST);
+        assign scalar_tmem_softmax_dispatch[i] = lsu_dispatch_if[i].valid
+                                             && (lsu_dispatch_if[i].data.op_type == `INST_LSU_TMEM_SOFTMAX);
+        assign scalar_tmem_dispatch[i] = scalar_tmem_ld_dispatch[i]
+                                      || scalar_tmem_st_dispatch[i]
+                                      || scalar_tmem_softmax_dispatch[i];
+
+        assign scalar_mem_lsu_dispatch_if[i].valid = lsu_dispatch_if[i].valid && !scalar_tmem_dispatch[i];
+        assign scalar_mem_lsu_dispatch_if[i].data = lsu_dispatch_if[i].data;
+        assign lsu_dispatch_if[i].ready = scalar_tmem_dispatch[i] ? scalar_tmem_dispatch_ready[i]
+                                                                  : scalar_mem_lsu_dispatch_if[i].ready;
+
+        assign scalar_tmem_dispatch_fire[i] = scalar_tmem_dispatch[i] && scalar_tmem_dispatch_ready[i];
+    end
+
     VX_mem_bus_if #(
         .DATA_SIZE (DCACHE_WORD_SIZE),
         .TAG_WIDTH (DCACHE_TAG_WIDTH)
@@ -301,10 +345,367 @@ module VX_execute import VX_gpu_pkg::*; #(
         .reset          (lsu_reset),
         .downstream_mem_busy (downstream_mem_busy),
         .cache_bus_if   (scalar_lsu_bus_if),
-        .dispatch_if    (lsu_dispatch_if),
-        .commit_if      (lsu_scalar_commit_if)
+        .dispatch_if    (scalar_mem_lsu_dispatch_if),
+        .commit_if      (scalar_mem_lsu_commit_if)
     );
 
+    wire scalar_tmem_pending;
+    reg  [`ISSUE_WIDTH-1:0] scalar_tmem_grant;
+    reg  [`ISSUE_WIDTH-1:0] scalar_tmem_grant_r;
+    reg                     scalar_tmem_load_pending;
+    reg                     scalar_tmem_store_pending;
+    reg                     scalar_tmem_commit_pending;
+    reg                     scalar_tmem_softmax_active;
+    reg                     scalar_tmem_softmax_read_pending;
+    reg [1:0]               scalar_tmem_softmax_stage;
+    reg [5:0]               scalar_tmem_softmax_index;
+    reg [5:0]               scalar_tmem_softmax_read_index_r;
+    reg [8:0]               scalar_tmem_softmax_score_base_r;
+    reg [8:0]               scalar_tmem_softmax_p_base_r;
+    reg [31:0]              scalar_tmem_softmax_scores_r [0:31][0:`NUM_THREADS-1];
+    reg [31:0]              scalar_tmem_softmax_row_max_r [0:`NUM_THREADS-1];
+    reg [31:0]              scalar_tmem_softmax_denom_r [0:`NUM_THREADS-1];
+    reg  [`UUID_WIDTH-1:0]  scalar_tmem_uuid_r;
+    reg  [`NW_WIDTH-1:0]    scalar_tmem_wid_r;
+    reg  [`NUM_THREADS-1:0] scalar_tmem_tmask_r;
+    reg  [`XLEN-1:0]        scalar_tmem_pc_r;
+    reg  [`NR_BITS-1:0]     scalar_tmem_rd_r;
+    reg  [`NUM_THREADS*`XLEN-1:0] scalar_tmem_rdata_r;
+    reg                     scalar_tmem_rdata_valid_r;
+    wire [`ISSUE_WIDTH-1:0][8:0] scalar_tmem_req_addr;
+    wire [`ISSUE_WIDTH-1:0][`NUM_THREADS*`XLEN-1:0] scalar_tmem_req_wdata;
+    wire [`ISSUE_WIDTH-1:0][`UUID_WIDTH-1:0] scalar_tmem_req_uuid;
+    wire [`ISSUE_WIDTH-1:0][`NW_WIDTH-1:0] scalar_tmem_req_wid;
+    wire [`ISSUE_WIDTH-1:0][`NUM_THREADS-1:0] scalar_tmem_req_tmask;
+    wire [`ISSUE_WIDTH-1:0][`XLEN-1:0] scalar_tmem_req_pc;
+    wire [`ISSUE_WIDTH-1:0][`NR_BITS-1:0] scalar_tmem_req_rd;
+    wire [`ISSUE_WIDTH-1:0][8:0] scalar_tmem_req_softmax_p_addr;
+
+    localparam [1:0] SCALAR_TMEM_SOFTMAX_MAX   = 2'd0;
+    localparam [1:0] SCALAR_TMEM_SOFTMAX_DENOM = 2'd1;
+    localparam [1:0] SCALAR_TMEM_SOFTMAX_WRITE = 2'd2;
+    localparam [5:0] SCALAR_TMEM_SOFTMAX_ROW_LAST  = 6'd31;
+    localparam [5:0] SCALAR_TMEM_SOFTMAX_TILE_LAST = 6'd63;
+
+    function automatic [31:0] scalar_tmem_f32_max;
+        input [31:0] a_bits;
+        input [31:0] b_bits;
+        begin
+        `ifdef SV_DPI
+            scalar_tmem_f32_max = 32'(dpi_f32_max(1'b1, int'(a_bits), int'(b_bits)));
+        `else
+            scalar_tmem_f32_max = a_bits;
+        `endif
+        end
+    endfunction
+
+    function automatic [31:0] scalar_tmem_softmax_exp_acc;
+        input [31:0] score_bits;
+        input [31:0] max_bits;
+        input [31:0] accum_bits;
+        begin
+        `ifdef SV_DPI
+            scalar_tmem_softmax_exp_acc = 32'(dpi_softmax_exp_acc(1'b1, int'(score_bits), int'(max_bits), int'(accum_bits)));
+        `else
+            scalar_tmem_softmax_exp_acc = accum_bits;
+        `endif
+        end
+    endfunction
+
+    function automatic [31:0] scalar_tmem_softmax_prob_pack;
+        input [31:0] score_bits;
+        input [31:0] max_bits;
+        input [31:0] denom_bits;
+        begin
+        `ifdef SV_DPI
+            scalar_tmem_softmax_prob_pack = 32'(dpi_softmax_prob_to_f16x2(1'b1, int'(score_bits), int'(max_bits), int'(denom_bits)));
+        `else
+            scalar_tmem_softmax_prob_pack = '0;
+        `endif
+        end
+    endfunction
+
+    assign scalar_tmem_pending = |scalar_tmem_dispatch;
+
+    always @(*) begin
+        scalar_tmem_grant = '0;
+        for (integer i = `ISSUE_WIDTH-1; i >= 0; --i) begin
+            if (scalar_tmem_dispatch[i]) begin
+                scalar_tmem_grant = '0;
+                scalar_tmem_grant[i] = 1'b1;
+            end
+        end
+    end
+
+    wire scalar_tmem_grant_valid = scalar_tmem_pending && !scalar_tmem_load_pending
+                                && !scalar_tmem_store_pending && !scalar_tmem_commit_pending
+                                && !scalar_tmem_softmax_active;
+    wire scalar_tmem_grant_is_load = |(scalar_tmem_grant & scalar_tmem_ld_dispatch);
+    wire scalar_tmem_grant_is_store = |(scalar_tmem_grant & scalar_tmem_st_dispatch);
+    wire scalar_tmem_grant_is_softmax = |(scalar_tmem_grant & scalar_tmem_softmax_dispatch);
+    wire scalar_tmem_req_ready = scalar_tmem_grant_is_softmax ? 1'b1 :
+                                 scalar_tmem_grant_is_load ? scalar_tmem_rready : scalar_tmem_wready;
+    wire scalar_tmem_req_fire = scalar_tmem_grant_valid && scalar_tmem_req_ready;
+    wire scalar_tmem_softmax_read_valid = scalar_tmem_softmax_active
+                                       && !scalar_tmem_softmax_read_pending
+                                       && (scalar_tmem_softmax_stage != SCALAR_TMEM_SOFTMAX_WRITE);
+    wire scalar_tmem_softmax_write_valid = scalar_tmem_softmax_active
+                                        && (scalar_tmem_softmax_stage == SCALAR_TMEM_SOFTMAX_WRITE);
+    wire scalar_tmem_softmax_read_issue = scalar_tmem_softmax_read_valid && scalar_tmem_rready;
+    wire scalar_tmem_softmax_write_issue = scalar_tmem_softmax_write_valid && scalar_tmem_wready;
+    wire [8:0] scalar_tmem_softmax_read_addr = scalar_tmem_softmax_score_base_r
+                                             + 9'(scalar_tmem_softmax_index[4:0]);
+    wire [8:0] scalar_tmem_softmax_write_addr = scalar_tmem_softmax_p_base_r
+                                              + 9'(scalar_tmem_softmax_index);
+    wire [`NUM_THREADS*`XLEN-1:0] scalar_tmem_softmax_wdata;
+    wire scalar_tmem_softmax_commit = scalar_tmem_commit_pending
+                                    && !scalar_tmem_load_pending
+                                    && !scalar_tmem_store_pending;
+    wire [`XLEN-1:0] scalar_tmem_softmax_token = `XLEN'(scalar_tmem_softmax_p_base_r);
+
+    for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin : g_scalar_tmem_ready
+        assign scalar_tmem_dispatch_ready[i] = scalar_tmem_grant_valid && scalar_tmem_grant[i] && scalar_tmem_req_ready;
+        assign scalar_tmem_req_addr[i] = lsu_dispatch_if[i].data.rs1_data[0][8:0];
+        assign scalar_tmem_req_wdata[i] = lsu_dispatch_if[i].data.rs2_data;
+        assign scalar_tmem_req_uuid[i] = lsu_dispatch_if[i].data.uuid;
+        assign scalar_tmem_req_wid[i] = wis_to_wid(lsu_dispatch_if[i].data.wis, ISSUE_ISW_W'(i));
+        assign scalar_tmem_req_tmask[i] = lsu_dispatch_if[i].data.tmask;
+        assign scalar_tmem_req_pc[i] = lsu_dispatch_if[i].data.PC;
+        assign scalar_tmem_req_rd[i] = lsu_dispatch_if[i].data.rd;
+        assign scalar_tmem_req_softmax_p_addr[i] = lsu_dispatch_if[i].data.rs2_data[0][8:0];
+
+        assign scalar_tmem_commit_if[i].valid = scalar_tmem_commit_pending && scalar_tmem_grant_r[i];
+        assign scalar_tmem_commit_if[i].data.uuid = scalar_tmem_uuid_r;
+        assign scalar_tmem_commit_if[i].data.wid = scalar_tmem_wid_r;
+        assign scalar_tmem_commit_if[i].data.tmask = scalar_tmem_tmask_r;
+        assign scalar_tmem_commit_if[i].data.PC = scalar_tmem_pc_r;
+        assign scalar_tmem_commit_if[i].data.wb = scalar_tmem_load_pending || scalar_tmem_softmax_commit;
+        assign scalar_tmem_commit_if[i].data.rd = (scalar_tmem_load_pending || scalar_tmem_softmax_commit) ? scalar_tmem_rd_r : '0;
+        assign scalar_tmem_commit_if[i].data.data =
+            scalar_tmem_load_pending
+                ? (scalar_tmem_rdata_valid_r ? scalar_tmem_rdata_r : scalar_tmem_rdata)
+                : {`NUM_THREADS{scalar_tmem_softmax_token}};
+        assign scalar_tmem_commit_if[i].data.tensor = 1'b0;
+        assign scalar_tmem_commit_if[i].data.pid = '0;
+        assign scalar_tmem_commit_if[i].data.sop = 1'b1;
+        assign scalar_tmem_commit_if[i].data.eop = 1'b1;
+        assign scalar_tmem_commit_ready[i] = scalar_tmem_commit_if[i].ready;
+    end
+
+    function automatic [`NUM_THREADS*`XLEN/8-1:0] scalar_tmem_expand_tmask;
+        input [`NUM_THREADS-1:0] tmask;
+        begin
+            scalar_tmem_expand_tmask = '0;
+            for (integer lane = 0; lane < `NUM_THREADS; ++lane) begin
+                scalar_tmem_expand_tmask[lane * (`XLEN / 8) +: (`XLEN / 8)] =
+                    {(`XLEN / 8){tmask[lane]}};
+            end
+        end
+    endfunction
+
+    for (genvar lane = 0; lane < `NUM_THREADS; ++lane) begin : g_scalar_tmem_softmax_wdata
+        assign scalar_tmem_softmax_wdata[lane * `XLEN +: `XLEN] =
+            scalar_tmem_softmax_prob_pack(
+                scalar_tmem_softmax_scores_r[scalar_tmem_softmax_index[4:0]][lane],
+                scalar_tmem_softmax_row_max_r[lane],
+                scalar_tmem_softmax_denom_r[lane]);
+    end
+
+`ifdef DBG_TRACE_CORE_PIPELINE_VCS
+    always @(posedge clk) begin
+        if (!reset && ($time > `TRACE_STARTTIME) && (CORE_ID == 0)) begin
+            if (scalar_tmem_req_fire && scalar_tmem_grant_is_softmax) begin
+                `TRACE(1, ("%d: core%0d-scalar-tmem-softmax-start: grant=%b wid=%0d PC=0x%0h rd=%0d score=%0d prob=%0d tmask=%b (#%0d)\n",
+                    $time, CORE_ID, scalar_tmem_grant, scalar_tmem_req_wid[0], scalar_tmem_req_pc[0],
+                    scalar_tmem_req_rd[0], scalar_tmem_req_addr[0], scalar_tmem_req_softmax_p_addr[0],
+                    scalar_tmem_req_tmask[0], scalar_tmem_req_uuid[0]));
+            end
+            if (scalar_tmem_softmax_read_issue && (scalar_tmem_softmax_index == 6'd0 || scalar_tmem_softmax_index == SCALAR_TMEM_SOFTMAX_ROW_LAST)) begin
+                `TRACE(1, ("%d: core%0d-scalar-tmem-softmax-read: stage=%0d index=%0d addr=%0d rready=%b PC=0x%0h rd=%0d\n",
+                    $time, CORE_ID, scalar_tmem_softmax_stage, scalar_tmem_softmax_index,
+                    scalar_tmem_softmax_read_addr, scalar_tmem_rready, scalar_tmem_pc_r, scalar_tmem_rd_r));
+            end
+            if (scalar_tmem_softmax_write_issue && (scalar_tmem_softmax_index == 6'd0 || scalar_tmem_softmax_index == SCALAR_TMEM_SOFTMAX_TILE_LAST)) begin
+                `TRACE(1, ("%d: core%0d-scalar-tmem-softmax-write: index=%0d addr=%0d wready=%b PC=0x%0h rd=%0d\n",
+                    $time, CORE_ID, scalar_tmem_softmax_index, scalar_tmem_softmax_write_addr,
+                    scalar_tmem_wready, scalar_tmem_pc_r, scalar_tmem_rd_r));
+            end
+            if (scalar_tmem_commit_pending && scalar_tmem_pc_r == 32'h80000318) begin
+                `TRACE(1, ("%d: core%0d-scalar-tmem-softmax-commit-pending: grant_r=%b ready=%b valid0=%b wb0=%0d rd=%0d PC=0x%0h token=0x%0h\n",
+                    $time, CORE_ID, scalar_tmem_grant_r, scalar_tmem_commit_ready,
+                    scalar_tmem_commit_if[0].valid, scalar_tmem_commit_if[0].data.wb,
+                    scalar_tmem_commit_if[0].data.rd, scalar_tmem_commit_if[0].data.PC,
+                    scalar_tmem_softmax_token));
+            end
+        end
+    end
+`endif
+
+    always @(*) begin
+        scalar_tmem_ren = (scalar_tmem_grant_valid && scalar_tmem_grant_is_load)
+                       || scalar_tmem_softmax_read_valid;
+        scalar_tmem_wen = (scalar_tmem_grant_valid && scalar_tmem_grant_is_store)
+                       || scalar_tmem_softmax_write_valid;
+        scalar_tmem_raddr = '0;
+        scalar_tmem_waddr = '0;
+        scalar_tmem_wdata = '0;
+        scalar_tmem_mask = '0;
+        for (integer i = 0; i < `ISSUE_WIDTH; ++i) begin
+            if (scalar_tmem_grant[i]) begin
+                scalar_tmem_raddr = scalar_tmem_req_addr[i];
+                scalar_tmem_waddr = scalar_tmem_req_addr[i];
+                scalar_tmem_wdata = scalar_tmem_req_wdata[i];
+                scalar_tmem_mask = scalar_tmem_expand_tmask(scalar_tmem_req_tmask[i])
+                                 & {(`NUM_THREADS * (`XLEN / 8)){scalar_tmem_grant_is_store}};
+            end
+        end
+        if (scalar_tmem_softmax_read_valid) begin
+            scalar_tmem_raddr = scalar_tmem_softmax_read_addr;
+        end
+        if (scalar_tmem_softmax_write_valid) begin
+            scalar_tmem_waddr = scalar_tmem_softmax_write_addr;
+            scalar_tmem_wdata = scalar_tmem_softmax_wdata;
+            scalar_tmem_mask = scalar_tmem_expand_tmask(scalar_tmem_tmask_r);
+        end
+    end
+
+    always @(posedge clk) begin
+        if (reset) begin
+            scalar_tmem_grant_r <= '0;
+            scalar_tmem_load_pending <= 1'b0;
+            scalar_tmem_store_pending <= 1'b0;
+            scalar_tmem_commit_pending <= 1'b0;
+            scalar_tmem_softmax_active <= 1'b0;
+            scalar_tmem_softmax_read_pending <= 1'b0;
+            scalar_tmem_softmax_stage <= SCALAR_TMEM_SOFTMAX_MAX;
+            scalar_tmem_softmax_index <= '0;
+            scalar_tmem_softmax_read_index_r <= '0;
+            scalar_tmem_softmax_score_base_r <= '0;
+            scalar_tmem_softmax_p_base_r <= '0;
+            scalar_tmem_uuid_r <= '0;
+            scalar_tmem_wid_r <= '0;
+            scalar_tmem_tmask_r <= '0;
+            scalar_tmem_pc_r <= '0;
+            scalar_tmem_rd_r <= '0;
+            scalar_tmem_rdata_r <= '0;
+            scalar_tmem_rdata_valid_r <= 1'b0;
+        end else begin
+            if (scalar_tmem_req_fire) begin
+                scalar_tmem_grant_r <= scalar_tmem_grant;
+                scalar_tmem_load_pending <= scalar_tmem_grant_is_load;
+                scalar_tmem_store_pending <= scalar_tmem_grant_is_store;
+                scalar_tmem_commit_pending <= !scalar_tmem_grant_is_softmax;
+                scalar_tmem_rdata_valid_r <= 1'b0;
+                for (integer i = 0; i < `ISSUE_WIDTH; ++i) begin
+                    if (scalar_tmem_grant[i]) begin
+                        scalar_tmem_uuid_r <= scalar_tmem_req_uuid[i];
+                        scalar_tmem_wid_r <= scalar_tmem_req_wid[i];
+                        scalar_tmem_tmask_r <= scalar_tmem_req_tmask[i];
+                        scalar_tmem_pc_r <= scalar_tmem_req_pc[i];
+                        scalar_tmem_rd_r <= scalar_tmem_req_rd[i];
+                        if (scalar_tmem_grant_is_softmax) begin
+                            scalar_tmem_softmax_active <= 1'b1;
+                            scalar_tmem_softmax_read_pending <= 1'b0;
+                            scalar_tmem_softmax_stage <= SCALAR_TMEM_SOFTMAX_MAX;
+                            scalar_tmem_softmax_index <= '0;
+                            scalar_tmem_softmax_read_index_r <= '0;
+                            scalar_tmem_softmax_score_base_r <= scalar_tmem_req_addr[i];
+                            scalar_tmem_softmax_p_base_r <= scalar_tmem_req_softmax_p_addr[i];
+                            for (integer lane = 0; lane < `NUM_THREADS; ++lane) begin
+                                scalar_tmem_softmax_row_max_r[lane] <= '0;
+                                scalar_tmem_softmax_denom_r[lane] <= '0;
+                            end
+                        end
+                    end
+                end
+            end else if (scalar_tmem_load_pending && scalar_tmem_commit_pending && !scalar_tmem_rdata_valid_r) begin
+                scalar_tmem_rdata_r <= scalar_tmem_rdata;
+                scalar_tmem_rdata_valid_r <= 1'b1;
+            end
+
+            if (scalar_tmem_softmax_read_issue) begin
+                scalar_tmem_softmax_read_pending <= 1'b1;
+                scalar_tmem_softmax_read_index_r <= scalar_tmem_softmax_index;
+            end else if (scalar_tmem_softmax_read_pending) begin
+                scalar_tmem_softmax_read_pending <= 1'b0;
+                for (integer lane = 0; lane < `NUM_THREADS; ++lane) begin
+                    scalar_tmem_softmax_scores_r[scalar_tmem_softmax_read_index_r[4:0]][lane]
+                        <= scalar_tmem_rdata[lane * `XLEN +: `XLEN];
+                    if (scalar_tmem_softmax_stage == SCALAR_TMEM_SOFTMAX_MAX) begin
+                        scalar_tmem_softmax_row_max_r[lane] <=
+                            (scalar_tmem_softmax_read_index_r == '0)
+                                ? scalar_tmem_rdata[lane * `XLEN +: `XLEN]
+                                : scalar_tmem_f32_max(scalar_tmem_softmax_row_max_r[lane],
+                                                      scalar_tmem_rdata[lane * `XLEN +: `XLEN]);
+                    end else if (scalar_tmem_softmax_stage == SCALAR_TMEM_SOFTMAX_DENOM) begin
+                        scalar_tmem_softmax_denom_r[lane] <=
+                            scalar_tmem_softmax_exp_acc(
+                                scalar_tmem_rdata[lane * `XLEN +: `XLEN],
+                                scalar_tmem_softmax_row_max_r[lane],
+                                scalar_tmem_softmax_denom_r[lane]);
+                    end
+                end
+
+                if (scalar_tmem_softmax_stage == SCALAR_TMEM_SOFTMAX_MAX) begin
+                    if (scalar_tmem_softmax_read_index_r == SCALAR_TMEM_SOFTMAX_ROW_LAST) begin
+                        scalar_tmem_softmax_stage <= SCALAR_TMEM_SOFTMAX_DENOM;
+                        scalar_tmem_softmax_index <= '0;
+                        for (integer lane = 0; lane < `NUM_THREADS; ++lane) begin
+                            scalar_tmem_softmax_denom_r[lane] <= 32'h00000000;
+                        end
+                    end else begin
+                        scalar_tmem_softmax_index <= scalar_tmem_softmax_read_index_r + 6'd1;
+                    end
+                end else if (scalar_tmem_softmax_stage == SCALAR_TMEM_SOFTMAX_DENOM) begin
+                    if (scalar_tmem_softmax_read_index_r == SCALAR_TMEM_SOFTMAX_ROW_LAST) begin
+                        scalar_tmem_softmax_stage <= SCALAR_TMEM_SOFTMAX_WRITE;
+                        scalar_tmem_softmax_index <= '0;
+                    end else begin
+                        scalar_tmem_softmax_index <= scalar_tmem_softmax_read_index_r + 6'd1;
+                    end
+                end
+            end
+
+            if (scalar_tmem_softmax_write_issue) begin
+                if (scalar_tmem_softmax_index == SCALAR_TMEM_SOFTMAX_TILE_LAST) begin
+                    scalar_tmem_softmax_active <= 1'b0;
+                    scalar_tmem_commit_pending <= 1'b1;
+                end else begin
+                    scalar_tmem_softmax_index <= scalar_tmem_softmax_index + 6'd1;
+                end
+            end
+
+            if (scalar_tmem_commit_pending && (|(scalar_tmem_grant_r & scalar_tmem_commit_ready))) begin
+                scalar_tmem_grant_r <= '0;
+                scalar_tmem_load_pending <= 1'b0;
+                scalar_tmem_store_pending <= 1'b0;
+                scalar_tmem_commit_pending <= 1'b0;
+                scalar_tmem_rdata_valid_r <= 1'b0;
+            end
+        end
+    end
+
+    localparam SCALAR_LSU_COMMIT_DATAW = `UUID_WIDTH + `NW_WIDTH + `NUM_THREADS + `XLEN + 1 + `NR_BITS + (`NUM_THREADS * `XLEN) + 1 + 1 + 1 + 1;
+
+    for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin : g_scalar_lsu_commit
+        VX_stream_arb #(
+            .NUM_INPUTS (2),
+            .DATAW      (SCALAR_LSU_COMMIT_DATAW),
+            .ARBITER    ("R"),
+            .OUT_REG    (1)
+        ) scalar_lsu_commit_arb (
+            .clk       (clk),
+            .reset     (reset),
+            .valid_in  ({scalar_tmem_commit_if[i].valid, scalar_mem_lsu_commit_if[i].valid}),
+            .ready_in  ({scalar_tmem_commit_if[i].ready, scalar_mem_lsu_commit_if[i].ready}),
+            .data_in   ({scalar_tmem_commit_if[i].data, scalar_mem_lsu_commit_if[i].data}),
+            .data_out  (lsu_scalar_commit_if[i].data),
+            .valid_out (lsu_scalar_commit_if[i].valid),
+            .ready_out (lsu_scalar_commit_if[i].ready),
+            `UNUSED_PIN (sel_out)
+        );
+    end
+
 `ifdef EXT_T_ENABLE
     VX_commit_if lsu_tensor_commit_if[`ISSUE_WIDTH]();
 

hw/rtl/core/VX_fpu_unit.sv

@@ -145,6 +145,7 @@ module VX_fpu_unit import VX_fpu_pkg::*; #(
 
             .valid_in   (fpu_req_valid),
             .op_type    (execute_if[block_idx].data.op_type),
+            .op_mod     (execute_if[block_idx].data.op_mod),
             .lane_mask  (execute_if[block_idx].data.tmask),
             .fmt        (fpu_fmt),
             .frm        (fpu_req_frm),
@@ -174,6 +175,7 @@ module VX_fpu_unit import VX_fpu_pkg::*; #(
 
             .valid_in   (fpu_req_valid),
             .op_type    (execute_if[block_idx].data.op_type),
+            .op_mod     (execute_if[block_idx].data.op_mod),
             .lane_mask  (execute_if[block_idx].data.tmask),
             .fmt        (fpu_fmt),
             .frm        (fpu_req_frm),
@@ -204,6 +206,7 @@ module VX_fpu_unit import VX_fpu_pkg::*; #(
             .valid_in   (fpu_req_valid),
             .lane_mask  (execute_if[block_idx].data.tmask),
             .op_type    (execute_if[block_idx].data.op_type),
+            .op_mod     (execute_if[block_idx].data.op_mod),
             .fmt        (fpu_fmt),
             .frm        (fpu_req_frm),
             .dataa      (execute_if[block_idx].data.rs1_data),

hw/rtl/core/VX_tensor_blackwell_core.sv

@@ -28,6 +28,9 @@ module VX_tensor_blackwell_core_block import VX_gpu_pkg::*; #(
     VX_tc_bus_if.master smem_B_if,
     VX_commit_if.master commit_if
 );
+    `STATIC_ASSERT((`NUM_THREADS == 4),
+        ("4-lane Blackwell tensor core wrapper requires NUM_THREADS == 4"))
+
     localparam NUM_LANES = `NUM_THREADS;
     localparam METADATA_QUEUE_DEPTH = 2;
 
@@ -144,10 +147,6 @@ module VX_tensor_blackwell_core_block import VX_gpu_pkg::*; #(
         .io_writeback_bits_data_1(writeback_data[1]),
         .io_writeback_bits_data_2(writeback_data[2]),
         .io_writeback_bits_data_3(writeback_data[3]),
-        .io_writeback_bits_data_4(writeback_data[4]),
-        .io_writeback_bits_data_5(writeback_data[5]),
-        .io_writeback_bits_data_6(writeback_data[6]),
-        .io_writeback_bits_data_7(writeback_data[7]),
 
         .io_respA_ready(tmem_if.rsp_ready),
         .io_respA_valid(tmem_if.rsp_valid),

hw/rtl/core/VX_trace.vh

@@ -327,6 +327,7 @@ task trace_ex_op(input int level,
             `INST_FPU_MISC: begin
                 if  (fdst_d) begin
                     case (op_mod)
+                        8: `TRACE(level, ("FEXP.D"));
                         0: `TRACE(level, ("FSGNJ.D"));
                         1: `TRACE(level, ("FSGNJN.D"));
                         2: `TRACE(level, ("FSGNJX.D"));
@@ -338,6 +339,7 @@ task trace_ex_op(input int level,
                     endcase
                 end else begin
                     case (op_mod)
+                        8: `TRACE(level, ("FEXP.S"));
                         0: `TRACE(level, ("FSGNJ.S"));
                         1: `TRACE(level, ("FSGNJN.S"));
                         2: `TRACE(level, ("FSGNJX.S"));

hw/rtl/fpu/VX_fpu_dpi.sv

@@ -31,6 +31,7 @@ module VX_fpu_dpi import VX_fpu_pkg::*; #(
     input wire [TAGW-1:0] tag_in,
     
     input wire [`INST_FPU_BITS-1:0] op_type,
+    input wire [`INST_MOD_BITS-1:0] op_mod,
     input wire [`INST_FMT_BITS-1:0] fmt,
     input wire [`INST_FRM_BITS-1:0] frm,
 
@@ -51,7 +52,8 @@ module VX_fpu_dpi import VX_fpu_pkg::*; #(
     localparam FPU_DIVSQRT = 1;
     localparam FPU_CVT     = 2;
     localparam FPU_NCP     = 3;
-    localparam NUM_FPC     = 4;
+    localparam FPU_EXP     = 4;
+    localparam NUM_FPC     = 5;
     localparam FPC_BITS    = `LOG2UP(NUM_FPC);
 
     localparam RSP_DATAW = (NUM_LANES * `XLEN) + 1 + $bits(fflags_t) + TAGW;
@@ -133,6 +135,7 @@ module VX_fpu_dpi import VX_fpu_pkg::*; #(
             `INST_FPU_I2F:   begin core_select = FPU_CVT; is_itof = 1; end
             `INST_FPU_U2F:   begin core_select = FPU_CVT; is_utof = 1; end
             `INST_FPU_F2F:   begin core_select = FPU_CVT; is_f2f  = 1; end            
+            `INST_FPU_MISC:  begin core_select = `INST_FPU_IS_EXP(op_type, op_mod) ? FPU_EXP : FPU_NCP; end
             default:         begin core_select = FPU_NCP; end
         endcase
     end
@@ -437,6 +440,45 @@ module VX_fpu_dpi import VX_fpu_pkg::*; #(
     end
     endgenerate
 
+    generate
+    begin : fexp
+
+        reg [NUM_LANES-1:0][`XLEN-1:0] result_fexp_r;
+        reg [NUM_LANES-1:0][63:0] result_fexp;
+        fflags_t [NUM_LANES-1:0] fflags_fexp;
+
+        wire fexp_valid = (valid_in && core_select == FPU_EXP);
+        wire fexp_ready = per_core_ready_out[FPU_EXP] || ~per_core_valid_out[FPU_EXP];
+        wire fexp_fire  = fexp_valid && fexp_ready;
+
+        always @(*) begin
+            for (integer i = 0; i < NUM_LANES; ++i) begin
+                dpi_fexp(fexp_fire, int'(dst_fmt), operands[0][i], result_fexp[i], fflags_fexp[i]);
+                result_fexp_r[i] = result_fexp[i][`XLEN-1:0];
+            end
+        end
+
+        fflags_t fflags_merged;
+        `FPU_MERGE_FFLAGS(fflags_merged, fflags_fexp, lane_mask, NUM_LANES);
+
+        VX_shift_register #(
+            .DATAW  (1 + TAGW + NUM_LANES * `XLEN + $bits(fflags_t)),
+            .DEPTH  (`LATENCY_FEXP),
+            .RESETW (1)
+        ) shift_reg (
+            .clk      (clk),
+            .reset    (reset),
+            .enable   (fexp_ready),
+            .data_in  ({fexp_valid, tag_in, result_fexp_r, fflags_merged}),
+            .data_out ({per_core_valid_out[FPU_EXP], per_core_tag_out[FPU_EXP], per_core_result[FPU_EXP], per_core_fflags[FPU_EXP]})
+        );
+
+        assign per_core_has_fflags[FPU_EXP] = 1;
+        assign per_core_ready_in[FPU_EXP] = fexp_ready;
+
+    end
+    endgenerate
+
     ///////////////////////////////////////////////////////////////////////////
 
     assign per_core_ready_in[FPU_DIVSQRT] = is_div ? div_ready_in : sqrt_ready_in;

hw/rtl/fpu/VX_fpu_dsp.sv

@@ -31,6 +31,7 @@ module VX_fpu_dsp import VX_fpu_pkg::*; #(
     input wire [TAGW-1:0] tag_in,
     
     input wire [`INST_FPU_BITS-1:0] op_type,
+    input wire [`INST_MOD_BITS-1:0] op_mod,
     input wire [`INST_FMT_BITS-1:0] fmt,
     input wire [`INST_FRM_BITS-1:0] frm,
 
@@ -51,7 +52,8 @@ module VX_fpu_dsp import VX_fpu_pkg::*; #(
     localparam FPU_DIVSQRT = 1;
     localparam FPU_CVT     = 2;
     localparam FPU_NCP     = 3;
-    localparam NUM_FPC     = 4;
+    localparam FPU_EXP     = 4;
+    localparam NUM_FPC     = 5;
     localparam FPC_BITS    = `LOG2UP(NUM_FPC);
 
     localparam RSP_DATAW = (NUM_LANES * 32) + 1 + $bits(fflags_t) + TAGW;
@@ -98,6 +100,7 @@ module VX_fpu_dsp import VX_fpu_pkg::*; #(
             `INST_FPU_F2U:    begin core_select = FPU_CVT; end
             `INST_FPU_I2F:    begin core_select = FPU_CVT; is_itof = 1; is_signed = 1; end
             `INST_FPU_U2F:    begin core_select = FPU_CVT; is_itof = 1; end
+            `INST_FPU_MISC:   begin core_select = `INST_FPU_IS_EXP(op_type, op_mod) ? FPU_EXP : FPU_NCP; end
             default:          begin core_select = FPU_NCP; end
         endcase
     end
@@ -107,6 +110,7 @@ module VX_fpu_dsp import VX_fpu_pkg::*; #(
     `RESET_RELAY (sqrt_reset, reset);
     `RESET_RELAY (cvt_reset, reset);
     `RESET_RELAY (ncp_reset, reset);
+    `RESET_RELAY (exp_reset, reset);
 
     wire [NUM_LANES-1:0][31:0] dataa_s;
     wire [NUM_LANES-1:0][31:0] datab_s;
@@ -243,6 +247,25 @@ module VX_fpu_dsp import VX_fpu_pkg::*; #(
         .ready_out  (per_core_ready_out[FPU_NCP])
     );
 
+    VX_fpu_exp #(
+        .NUM_LANES (NUM_LANES),
+        .TAGW      (TAGW)
+    ) fpu_exp (
+        .clk        (clk),
+        .reset      (exp_reset),
+        .valid_in   (valid_in && (core_select == FPU_EXP)),
+        .ready_in   (per_core_ready_in[FPU_EXP]),
+        .lane_mask  (lane_mask),
+        .tag_in     (tag_in),
+        .dataa      (dataa_s),
+        .has_fflags (per_core_has_fflags[FPU_EXP]),
+        .fflags     (per_core_fflags[FPU_EXP]),
+        .result     (per_core_result[FPU_EXP]),
+        .tag_out    (per_core_tag_out[FPU_EXP]),
+        .valid_out  (per_core_valid_out[FPU_EXP]),
+        .ready_out  (per_core_ready_out[FPU_EXP])
+    );
+
     ///////////////////////////////////////////////////////////////////////////
 
     assign per_core_ready_in[FPU_DIVSQRT] = is_div ? div_ready_in : sqrt_ready_in;

hw/rtl/fpu/VX_fpu_exp.sv

@@ -0,0 +1,92 @@
+// Copyright © 2019-2023
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+`include "VX_fpu_define.vh"
+`ifdef SV_DPI
+`include "float_dpi.vh"
+`else
+`ERROR(("VX_fpu_exp requires SV_DPI; replace dpi_fexp with synthesizable exp RTL for synthesis"))
+`endif
+
+module VX_fpu_exp import VX_fpu_pkg::*; #(
+    parameter NUM_LANES = 1,
+    parameter TAGW = 1
+) (
+    input wire clk,
+    input wire reset,
+
+    output wire ready_in,
+    input wire  valid_in,
+
+    input wire [NUM_LANES-1:0] lane_mask,
+
+    input wire [TAGW-1:0] tag_in,
+
+    input wire [NUM_LANES-1:0][31:0] dataa,
+    output wire [NUM_LANES-1:0][31:0] result,
+
+    output wire has_fflags,
+    output wire [`FP_FLAGS_BITS-1:0] fflags,
+
+    output wire [TAGW-1:0] tag_out,
+
+    input wire  ready_out,
+    output wire valid_out
+);
+    wire stall  = ~ready_out && valid_out;
+    wire enable = ~stall;
+
+    fflags_t [NUM_LANES-1:0] per_lane_fflags;
+    wire [NUM_LANES-1:0] lane_mask_out;
+
+    VX_shift_register #(
+        .DATAW  (1 + NUM_LANES + TAGW),
+        .DEPTH  (`LATENCY_FEXP),
+        .RESETW (1)
+    ) shift_reg (
+        .clk      (clk),
+        .reset    (reset),
+        .enable   (enable),
+        .data_in  ({valid_in, lane_mask, tag_in}),
+        .data_out ({valid_out, lane_mask_out, tag_out})
+    );
+
+    assign ready_in = enable;
+
+    for (genvar i = 0; i < NUM_LANES; ++i) begin
+        reg [63:0] r;
+        `UNUSED_VAR (r)
+
+        fflags_t f;
+
+        always @(*) begin
+            dpi_fexp(enable && valid_in, int'(0), {32'hffffffff, dataa[i]}, r, f);
+        end
+
+        VX_shift_register #(
+            .DATAW  (32 + $bits(fflags_t)),
+            .DEPTH  (`LATENCY_FEXP)
+        ) shift_req_dpi (
+            .clk      (clk),
+            `UNUSED_PIN (reset),
+            .enable   (enable),
+            .data_in  ({r[31:0], f}),
+            .data_out ({result[i], per_lane_fflags[i]})
+        );
+    end
+
+    assign has_fflags = 1;
+
+    `FPU_MERGE_FFLAGS(fflags, per_lane_fflags, lane_mask_out, NUM_LANES);
+
+endmodule

hw/rtl/fpu/VX_fpu_fpnew.sv

@@ -34,6 +34,7 @@ module VX_fpu_fpnew
     input wire [TAGW-1:0] tag_in,
     
     input wire [`INST_FPU_BITS-1:0] op_type,
+    input wire [`INST_MOD_BITS-1:0] op_mod,
     input wire [`INST_FMT_BITS-1:0] fmt,
     input wire [`INST_FRM_BITS-1:0] frm,
 
@@ -104,6 +105,7 @@ module VX_fpu_fpnew
     reg fpu_has_fflags, fpu_has_fflags_out;
     fpnew_pkg::fp_format_e fpu_src_fmt, fpu_dst_fmt;
     fpnew_pkg::int_format_e fpu_int_fmt;
+    wire is_fexp = `INST_FPU_IS_EXP(op_type, op_mod);
 
     `UNUSED_VAR (fmt)
 
@@ -183,7 +185,6 @@ module VX_fpu_fpnew
     end
 
 `ifdef XLEN_64
-    `UNUSED_VAR (lane_mask)
     for (genvar i = 0; i < NUM_LANES; ++i) begin
         wire [(TAGW+1)-1:0] fpu_tag;        
         wire fpu_valid_out_uq;
@@ -210,7 +211,7 @@ module VX_fpu_fpnew
             .int_fmt_i      (fpu_int_fmt),
             `UNUSED_PIN (vectorial_op_i),
             `UNUSED_PIN (simd_mask_i),
-            .tag_i          ({fpu_tag_in, fpu_has_fflags}),            
+            .tag_i          ({fpu_tag_in, fpu_has_fflags}),
             .in_valid_i     (fpu_valid_in),
             .in_ready_o     (fpu_ready_in_uq),
             .flush_i        (reset),
@@ -248,8 +249,8 @@ module VX_fpu_fpnew
         .int_fmt_i      (fpu_int_fmt),
         .vectorial_op_i (1'b1),
         .simd_mask_i    (lane_mask),
-        .tag_i          ({fpu_tag_in, fpu_has_fflags}),        
-        .in_valid_i     (fpu_valid_in),
+            .tag_i          ({fpu_tag_in, fpu_has_fflags}),
+            .in_valid_i     (fpu_valid_in),
         .in_ready_o     (fpu_ready_in),
         .flush_i        (reset),
         .result_o       (fpu_result),
@@ -261,10 +262,70 @@ module VX_fpu_fpnew
     );
 `endif
 
-    assign fpu_valid_in = valid_in;
-    assign ready_in = fpu_ready_in;
+    wire exp_ready_in;
+    wire exp_valid_out;
+    wire [NUM_LANES-1:0][31:0] exp_dataa;
+    wire [NUM_LANES-1:0][31:0] exp_result_s;
+    wire [NUM_LANES-1:0][`XLEN-1:0] exp_result;
+    wire exp_has_fflags;
+    fflags_t exp_fflags;
+    wire [TAGW-1:0] exp_tag_out;
+    wire exp_ready_out;
+
+    for (genvar i = 0; i < NUM_LANES; ++i) begin
+        assign exp_dataa[i] = dataa[i][31:0];
+    `ifdef FPU_RV64F
+        assign exp_result[i] = {32'hffffffff, exp_result_s[i]};
+    `else
+        assign exp_result[i] = exp_result_s[i];
+    `endif
+    end
+
+    VX_fpu_exp #(
+        .NUM_LANES (NUM_LANES),
+        .TAGW      (TAGW)
+    ) fpu_exp (
+        .clk        (clk),
+        .reset      (reset),
+        .valid_in   (valid_in && is_fexp),
+        .ready_in   (exp_ready_in),
+        .lane_mask  (lane_mask),
+        .tag_in     (tag_in),
+        .dataa      (exp_dataa),
+        .has_fflags (exp_has_fflags),
+        .fflags     (exp_fflags),
+        .result     (exp_result_s),
+        .tag_out    (exp_tag_out),
+        .valid_out  (exp_valid_out),
+        .ready_out  (exp_ready_out)
+    );
+
+    assign fpu_valid_in = valid_in && !is_fexp;
+    assign ready_in = is_fexp ? exp_ready_in : fpu_ready_in;
     assign fpu_tag_in = tag_in;
 
+    wire [RSP_DATAW-1:0] exp_rsp_data;
+    wire exp_rsp_valid;
+    wire exp_rsp_ready;
+    wire [RSP_DATAW-1:0] fpu_rsp_data;
+    wire fpu_rsp_valid;
+    wire fpu_rsp_ready;
+
+    VX_elastic_buffer #(
+        .DATAW   (RSP_DATAW),
+        .SIZE    (`OUT_REG_TO_EB_SIZE(OUT_REG)),
+        .OUT_REG (`OUT_REG_TO_EB_REG(OUT_REG))
+    ) exp_rsp_buf (
+        .clk       (clk),
+        .reset     (reset),
+        .valid_in  (exp_valid_out),
+        .ready_in  (exp_ready_out),
+        .data_in   ({exp_result, exp_has_fflags, exp_fflags, exp_tag_out}),
+        .data_out  (exp_rsp_data),
+        .valid_out (exp_rsp_valid),
+        .ready_out (exp_rsp_ready)
+    );
+
     VX_elastic_buffer #(
         .DATAW   (RSP_DATAW),
         .SIZE    (`OUT_REG_TO_EB_SIZE(OUT_REG)),
@@ -275,9 +336,26 @@ module VX_fpu_fpnew
         .valid_in  (fpu_valid_out),
         .ready_in  (fpu_ready_out),
         .data_in   ({fpu_result, fpu_has_fflags_out, fpu_status, fpu_tag_out}),
+        .data_out  (fpu_rsp_data),
+        .valid_out (fpu_rsp_valid),
+        .ready_out (fpu_rsp_ready)
+    );
+
+    VX_stream_arb #(
+        .NUM_INPUTS (2),
+        .DATAW      (RSP_DATAW),
+        .ARBITER    ("R"),
+        .OUT_REG    (0)
+    ) rsp_arb (
+        .clk       (clk),
+        .reset     (reset),
+        .valid_in  ({exp_rsp_valid, fpu_rsp_valid}),
+        .ready_in  ({exp_rsp_ready, fpu_rsp_ready}),
+        .data_in   ({exp_rsp_data, fpu_rsp_data}),
         .data_out  ({result, has_fflags, fflags, tag_out}),
         .valid_out (valid_out),
-        .ready_out (ready_out)
+        .ready_out (ready_out),
+        `UNUSED_PIN (sel_out)
     );
 
 endmodule

kernel/include/VX_config.h

@@ -402,6 +402,11 @@
 #endif
 #endif
 
+// FEXP Latency
+#ifndef LATENCY_FEXP
+#define LATENCY_FEXP 8
+#endif
+
 // FCVT Latency
 #ifndef LATENCY_FCVT
 #define LATENCY_FCVT 5