Merge branch 'graphics' of https://github.com/hansungk/rocket-chip into graphics

src/main/resources/csrc/SimMemTrace.cc

@@ -193,7 +193,7 @@ extern "C" void memtrace_query(unsigned char trace_read_ready,
                                unsigned char *trace_read_valid,
                                unsigned long *trace_read_address,
                                unsigned char *trace_read_is_store,
-                               int           *trace_read_size,
+                               unsigned char *trace_read_size, // logsize, don't need full int
                                unsigned long *trace_read_data,
                                unsigned char *trace_read_finished) {
   // printf("memtrace_query(cycle=%ld, tid=%d)\n", trace_read_cycle,

src/main/resources/csrc/SimMemTrace.h

@@ -53,7 +53,7 @@ extern "C" void memtrace_query(unsigned char trace_read_ready,
                                unsigned char *trace_read_valid,
                                unsigned long *trace_read_address,
                                unsigned char *trace_read_is_store,
-                               int           *trace_read_size,
+                               unsigned char *trace_read_size,
                                unsigned long *trace_read_data,
                                unsigned char *trace_read_finished);
 extern "C" int memtracelogger_init(int is_response, const char *filename,

src/main/resources/vsrc/SimMemTrace.v

@@ -1,7 +1,7 @@
 // FIXME hardcoded
-`define DATA_WIDTH 64
+`define MEMTRACE_DATA_WIDTH 64
 `define MAX_NUM_LANES 32
-`define LOGSIZE_WIDTH 8
+`define MEMTRACE_LOGSIZE_WIDTH 8
 
 import "DPI-C" function void memtrace_init(
   input  string  filename,
@@ -20,7 +20,7 @@ import "DPI-C" function void memtrace_query
   output bit     trace_read_valid,
   output longint trace_read_address,
   output bit     trace_read_is_store,
-  output int     trace_read_size,
+  output byte    trace_read_size,
   output longint trace_read_data,
   output bit     trace_read_finished
 );
@@ -36,16 +36,16 @@ module SimMemTrace #(parameter FILENAME = "undefined",
   // These have to match the IO port name of the Chisel wrapper module.
   input                                 trace_read_ready,
   output [NUM_LANES-1:0]                trace_read_valid,
-  output [`DATA_WIDTH*NUM_LANES-1:0]    trace_read_address,
+  output [`MEMTRACE_DATA_WIDTH*NUM_LANES-1:0]    trace_read_address,
   output [NUM_LANES-1:0]                trace_read_is_store,
-  output [`LOGSIZE_WIDTH*NUM_LANES-1:0] trace_read_size,
+  output [`MEMTRACE_LOGSIZE_WIDTH*NUM_LANES-1:0] trace_read_size,
-  output [`DATA_WIDTH*NUM_LANES-1:0]    trace_read_data,
+  output [`MEMTRACE_DATA_WIDTH*NUM_LANES-1:0]    trace_read_data,
   output                                trace_read_finished
 );
   bit                      __in_valid   [NUM_LANES-1:0];
   longint                  __in_address [NUM_LANES-1:0];
   bit                      __in_is_store [NUM_LANES-1:0];
-  reg [`LOGSIZE_WIDTH-1:0] __in_size [NUM_LANES-1:0];
+  reg [`MEMTRACE_LOGSIZE_WIDTH-1:0] __in_size [NUM_LANES-1:0];
   longint                  __in_data [NUM_LANES-1:0];
   bit                      __in_finished;
 
@@ -53,11 +53,11 @@ module SimMemTrace #(parameter FILENAME = "undefined",
   generate
     for (g = 0; g < NUM_LANES; g = g + 1) begin
       assign trace_read_valid[g] = __in_valid[g];
-      assign trace_read_address[`DATA_WIDTH*(g+1)-1:`DATA_WIDTH*g]  = __in_address[g];
+      assign trace_read_address[`MEMTRACE_DATA_WIDTH*(g+1)-1:`MEMTRACE_DATA_WIDTH*g]  = __in_address[g];
 
       assign trace_read_is_store[g] = __in_is_store[g];
-      assign trace_read_size[`LOGSIZE_WIDTH*(g+1)-1:`LOGSIZE_WIDTH*g] = __in_size[g];
+      assign trace_read_size[`MEMTRACE_LOGSIZE_WIDTH*(g+1)-1:`MEMTRACE_LOGSIZE_WIDTH*g] = __in_size[g];
-      assign trace_read_data[`DATA_WIDTH*(g+1)-1:`DATA_WIDTH*g] = __in_data[g];
+      assign trace_read_data[`MEMTRACE_DATA_WIDTH*(g+1)-1:`MEMTRACE_DATA_WIDTH*g] = __in_data[g];
     end
   endgenerate
   assign trace_read_finished = __in_finished;
@@ -71,11 +71,11 @@ module SimMemTrace #(parameter FILENAME = "undefined",
     if (reset) begin
       for (integer tid = 0; tid < NUM_LANES; tid = tid + 1) begin
         __in_valid[tid] = 1'b0;
-        __in_address[tid] = `DATA_WIDTH'b0;
+        __in_address[tid] = `MEMTRACE_DATA_WIDTH'b0;
         
         __in_is_store[tid] = 1'b0;
-        __in_size[tid] = `LOGSIZE_WIDTH'b0;
+        __in_size[tid] = `MEMTRACE_LOGSIZE_WIDTH'b0;
-        __in_data[tid] = `DATA_WIDTH'b0;
+        __in_data[tid] = `MEMTRACE_DATA_WIDTH'b0;
       end
       __in_finished = 1'b0;
     end else begin

src/main/scala/rocket/VortexCore.scala

@@ -12,8 +12,44 @@ import freechips.rocketchip.util._
 import freechips.rocketchip.scie._
 import tile.VortexTile
 
+class VortexBundle(tile: VortexTile)(implicit p: Parameters) extends CoreBundle {
+  val clock = Input(Clock())
+  val reset = Input(Reset())
+  // val hartid = Input(UInt(hartIdLen.W))
+  val reset_vector = Input(UInt(resetVectorLen.W))
+  val interrupts = Input(new CoreInterrupts())
+  
+  // conditionally instantiate ports depending on whether we want to use VX_cache or not
+  val imem = if (!tile.vortexParams.useVxCache) Some(Vec(1, new Bundle { // TODO: magic number
+    val a = tile.imemNodes.head.out.head._1.a.cloneType
+    val d = Flipped(tile.imemNodes.head.out.head._1.d.cloneType)
+  })) else None
+  val dmem = if (!tile.vortexParams.useVxCache) Some(Vec(4, new Bundle {
+    val a = tile.dmemNodes.head.out.head._1.a.cloneType
+    val d = Flipped(tile.dmemNodes.head.out.head._1.d.cloneType)
+  })) else None
+  val mem = if (tile.vortexParams.useVxCache) Some(new Bundle { 
+    val a = tile.memNode.out.head._1.a.cloneType
+    val d = Flipped(tile.memNode.out.head._1.d.cloneType)
+  }) else None
+
+  // val fpu = Flipped(new FPUCoreIO())
+  //val rocc = Flipped(new RoCCCoreIO(nTotalRoCCCSRs))
+  //val trace = Output(new TraceBundle)
+  //val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth)))
+  val cease = Output(Bool())
+  val wfi = Output(Bool())
+  val traceStall = Input(Bool())
+}
+
 class Vortex(tile: VortexTile)(implicit p: Parameters)
-    extends BlackBox with HasBlackBoxResource {
+    extends BlackBox(
+      // Each Vortex core gets tied-off hartId of 0, 1, 2, 3, ...
+      // The actual MHARTID read by the program is different by warp, not core;
+      // see VX_csr_data that implements the read logic for CSR_MHARTID/GWID.
+      Map("CORE_ID" -> tile.tileParams.hartId)
+    )
+    with HasBlackBoxResource {
   // addResource("/vsrc/vortex/hw/unit_tests/generic_queue/testbench.v")
   // addResource("/vsrc/vortex/hw/unit_tests/VX_divide_tb.v")
   // addResource("/vsrc/vortex/hw/syn/synopsys/models/memory/cln28hpm/rf2_256x19_wm0/rf2_256x19_wm0_rtl.v")
@@ -33,17 +69,9 @@ class Vortex(tile: VortexTile)(implicit p: Parameters)
   // addResource("/vsrc/vortex/hw/syn/modelsim/vortex_tb.v")
   addResource("/vsrc/vortex/hw/rtl/VX_dispatch.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_issue.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_shared_mem.sv")
+
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_core_rsp_merge.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_tag_access.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_core_req_bank_sel.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_bank.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_cache.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_data_access.sv")
   addResource("/vsrc/vortex/hw/rtl/cache/VX_cache_define.vh")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_flush_ctrl.sv")
+  
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_nc_bypass.sv")
-  // addResource("/vsrc/vortex/hw/rtl/cache/VX_miss_resrv.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_warp_sched.sv")
   // addResource("/vsrc/vortex/hw/rtl/Vortex.sv")
   addResource("/vsrc/vortex/hw/rtl/tex_unit/VX_tex_sat.sv")
@@ -72,7 +100,6 @@ class Vortex(tile: VortexTile)(implicit p: Parameters)
   // unused addResource("/vsrc/vortex/hw/rtl/libs/VX_mux.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_lzc.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_fifo_queue.sv")
-  // unused addResource("/vsrc/vortex/hw/rtl/libs/VX_scope.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_scan.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_find_first.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_multiplier.sv")
@@ -98,7 +125,7 @@ class Vortex(tile: VortexTile)(implicit p: Parameters)
   // unused addResource("/vsrc/vortex/hw/rtl/libs/VX_bypass_buffer.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_sp_ram.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_stream_demux.sv")
-  // addResource("/vsrc/vortex/hw/rtl/libs/VX_pending_size.sv")
+  
   // unused addResource("/vsrc/vortex/hw/rtl/libs/VX_index_queue.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_serial_div.sv")
   addResource("/vsrc/vortex/hw/rtl/libs/VX_fair_arbiter.sv")
@@ -110,13 +137,11 @@ class Vortex(tile: VortexTile)(implicit p: Parameters)
   addResource("/vsrc/vortex/hw/rtl/VX_execute.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_fetch.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_alu_unit.sv")
-  // unused addResource("/vsrc/vortex/hw/rtl/VX_mem_arb.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_platform.vh")
   addResource("/vsrc/vortex/hw/rtl/VX_commit.sv")
-  // unused addResource("/vsrc/vortex/hw/rtl/VX_smem_arb.sv")
+  
   addResource("/vsrc/vortex/hw/rtl/VX_pipeline.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_lsu_unit.sv")
-  // addResource("/vsrc/vortex/hw/rtl/VX_mem_unit.sv")
   addResource("/vsrc/vortex/hw/rtl/VX_csr_unit.sv")
   // addResource("/vsrc/vortex/hw/rtl/Vortex_axi.sv")
   // addResource("/vsrc/vortex/hw/rtl/fp_cores/VX_fp_div.sv")
@@ -196,30 +221,28 @@ class Vortex(tile: VortexTile)(implicit p: Parameters)
   // addResource("/vsrc/vortex/hw/rtl/afu/VX_avs_wrapper.sv")
   // addResource("/vsrc/vortex/hw/rtl/afu/VX_to_mem.sv")
   // addResource("/vsrc/vortex/sim/vlsim/vortex_afu_shim.sv")
-  addResource("/vsrc/vortex/hw/rtl/VX_pipeline_wrapper.sv")
+  if (tile.vortexParams.useVxCache) {
-
+    addResource("/vsrc/vortex/hw/rtl/libs/VX_pending_size.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_shared_mem.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_core_rsp_merge.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_tag_access.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_core_req_bank_sel.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_bank.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_data_access.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_flush_ctrl.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_nc_bypass.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_miss_resrv.sv")
+    addResource("/vsrc/vortex/hw/rtl/cache/VX_cache.sv")
+    addResource("/vsrc/vortex/hw/rtl/VX_mem_arb.sv")
+    addResource("/vsrc/vortex/hw/rtl/VX_smem_arb.sv")
+    addResource("/vsrc/vortex/hw/rtl/VX_mem_unit.sv")
+    addResource("/vsrc/vortex/hw/rtl/VX_core.sv")
+    addResource("/vsrc/vortex/hw/rtl/VX_core_wrapper.sv")
+  } else {
+    addResource("/vsrc/vortex/hw/rtl/VX_pipeline_wrapper.sv")
+  }
 
   val nTotalRoCCCSRs = 0
-  val io = IO(new CoreBundle()(p) {
+  val coreBundle = new VortexBundle(tile)
-    val clock = Input(Clock())
+  val io = IO(coreBundle)
-    val reset = Input(Reset())
-    val hartid = Input(UInt(hartIdLen.W))
-    val reset_vector = Input(UInt(resetVectorLen.W))
-    val interrupts = Input(new CoreInterrupts())
-    val imem = Vec(1, new Bundle { // TODO: magic number
-      val a = tile.imemNodes.head.out.head._1.a.cloneType
-      val d = Flipped(tile.imemNodes.head.out.head._1.d.cloneType)
-    })
-    val dmem = Vec(4, new Bundle {
-      val a = tile.dmemNodes.head.out.head._1.a.cloneType
-      val d = Flipped(tile.dmemNodes.head.out.head._1.d.cloneType)
-    })
-    val fpu = Flipped(new FPUCoreIO())
-    //val rocc = Flipped(new RoCCCoreIO(nTotalRoCCCSRs))
-    //val trace = Output(new TraceBundle)
-    //val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth)))
-    val cease = Output(Bool())
-    val wfi = Output(Bool())
-    val traceStall = Input(Bool())
-  })
 }

src/main/scala/tile/VortexTile.scala

@@ -24,22 +24,9 @@ case class RocketTileBoundaryBufferParams(force: Boolean = false)
 
 case class VortexTileParams(
     core: RocketCoreParams = RocketCoreParams(),
-    icache: Option[ICacheParams] = Some(ICacheParams(
+    useVxCache: Boolean = false,
-      nSets = 64,
+    icache: Option[ICacheParams] = None /* Some(ICacheParams()) */,
-      nWays = 4,
+    dcache: Option[DCacheParams] = None /* Some(DCacheParams()) */,
-      rowBits = 128,
-      nTLBSets = 1,
-      nTLBWays = 32,
-      nTLBBasePageSectors = 4,
-      nTLBSuperpages = 4,
-      cacheIdBits = 0,
-      blockBytes = 64,
-      latency = 2,
-      fetchBytes = 4
-    )),
-    dcache: Option[DCacheParams] = Some(DCacheParams(
-      // TODO
-    )),
     btb: Option[BTBParams] = None, // Some(BTBParams()),
     dataScratchpadBytes: Int = 0,
     name: Option[String] = Some("vortex_tile"),
@@ -49,8 +36,9 @@ case class VortexTileParams(
     clockSinkParams: ClockSinkParameters = ClockSinkParameters(),
     boundaryBuffers: Option[RocketTileBoundaryBufferParams] = None
     ) extends InstantiableTileParams[VortexTile] {
-  require(icache.isDefined)
+  // require(icache.isDefined)
-  require(dcache.isDefined)
+  // require(dcache.isDefined) 
+
   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): VortexTile = {
     new VortexTile(this, crossing, lookup)
   }
@@ -121,8 +109,24 @@ class VortexTile private(
     ))
   )))}
 
-  imemNodes.foreach { tlMasterXbar.node := _ }
+  val memNode = TLClientNode(Seq(TLMasterPortParameters.v1(
-  dmemNodes.foreach { tlMasterXbar.node := _ }
+    clients = Seq(TLMasterParameters.v1(
+      sourceId = IdRange(0, 1 << 15), // TODO magic numbers
+      name = s"Vortex Core ${vortexParams.hartId} Mem Interface",
+      requestFifo = true,
+      supportsProbe = TransferSizes(16, 16),
+      supportsGet = TransferSizes(16, 16),
+      supportsPutFull = TransferSizes(16, 16),
+      supportsPutPartial = TransferSizes(16, 16)
+    )),
+  )))
+  
+  if (vortexParams.useVxCache) {
+    tlMasterXbar.node := TLWidthWidget(16) := memNode
+  } else {
+    imemNodes.foreach { tlMasterXbar.node := _ }
+    dmemNodes.foreach { tlMasterXbar.node := _ }
+  }
 
   /* below are copied from rocket */
 
@@ -184,7 +188,7 @@ class VortexTileModuleImp(outer: VortexTile) extends BaseTileModuleImp(outer) {
   Annotated.params(this, outer.vortexParams)
 
   val core = Module(new Vortex(outer)(outer.p))
-
+  
   core.io.clock := clock
   core.io.reset := reset
 
@@ -213,42 +217,51 @@ class VortexTileModuleImp(outer: VortexTile) extends BaseTileModuleImp(outer) {
   // outer.traceSourceNode.bundle <> core.io.trace
   core.io.traceStall := outer.traceAuxSinkNode.bundle.stall
   // outer.bpwatchSourceNode.bundle <> core.io.bpwatch
-  core.io.hartid := outer.hartIdSinkNode.bundle
-  require(core.io.hartid.getWidth >= outer.hartIdSinkNode.bundle.getWidth,
-    s"core hartid wire (${core.io.hartid.getWidth}b) truncates external hartid wire (${outer.hartIdSinkNode.bundle.getWidth}b)")
 
-  (core.io.imem zip outer.imemNodes).foreach { case (coreMem, tileNode) =>
+  // Copypasted from Rocket; not necessary for Vortex as hartId is set via Verilog parameter
-    coreMem.d <> tileNode.out.head._1.d
+  // core.io.hartid := outer.hartIdSinkNode.bundle
-    coreMem.a <> tileNode.out.head._1.a
+  // require(core.io.hartid.getWidth >= outer.hartIdSinkNode.bundle.getWidth,
+  //   s"core hartid wire (${core.io.hartid.getWidth}b) truncates external hartid wire (${outer.hartIdSinkNode.bundle.getWidth}b)")
+
+  if (outer.vortexParams.useVxCache) {
+    println(s"width of a channel data ${core.io.mem.get.a.bits.data.getWidth}")
+    println(s"width of d channel data ${core.io.mem.get.d.bits.data.getWidth}")
+    core.io.mem.get.a <> outer.memNode.out.head._1.a
+    core.io.mem.get.d <> outer.memNode.out.head._1.d
+  }
+  else {
+    (core.io.imem.get zip outer.imemNodes).foreach { case (coreMem, tileNode) =>
+      coreMem.d <> tileNode.out.head._1.d
+      coreMem.a <> tileNode.out.head._1.a
+    }
+
+    // pick source id and:
+    // - lie to core that response is not valid if source doesn't match picked
+    // - lie to downstream that core is not ready if source doesn't match picked
+
+    val arb = Module(new RRArbiter(core.io.dmem.get.head.d.bits.source.cloneType, 4))
+    val matchingSources = Wire(UInt(4.W))
+    val dmemDs = outer.dmemNodes.map(_.out.head._1.d)
+
+    (arb.io.in zip dmemDs).zipWithIndex.foreach { case ((arbIn, tileNode), i) =>
+      arbIn.valid := tileNode.valid
+      arbIn.bits := tileNode.bits.source
+    }
+    matchingSources := dmemDs.map(d => (d.bits.source === arb.io.out.bits) && arb.io.out.valid).asUInt
+    arb.io.out.ready := true.B
+
+    (core.io.dmem.get zip dmemDs).zipWithIndex.foreach { case ((coreMem, tileNode), i) =>
+      coreMem.d.bits := tileNode.bits
+      coreMem.d.valid := tileNode.valid && matchingSources(i)
+      tileNode.ready := coreMem.d.ready && matchingSources(i)
+    }
+
+    (core.io.dmem.get zip outer.dmemNodes).foreach { case (coreMem, tileNode) =>
+      coreMem.a <> tileNode.out.head._1.a
+    }
   }
 
-
+  // core.io.fpu := DontCare
-  // pick source id and:
-  // - lie to core that response is not valid if source doesn't match picked
-  // - lie to downstream that core is not ready if source doesn't match picked
-
-  val arb = Module(new RRArbiter(core.io.dmem.head.d.bits.source.cloneType, 4))
-  val matchingSources = Wire(UInt(4.W))
-  val dmemDs = outer.dmemNodes.map(_.out.head._1.d)
-
-  (arb.io.in zip dmemDs).zipWithIndex.foreach { case ((arbIn, tileNode), i) =>
-    arbIn.valid := tileNode.valid
-    arbIn.bits := tileNode.bits.source
-  }
-  matchingSources := dmemDs.map(d => (d.bits.source === arb.io.out.bits) && arb.io.out.valid).asUInt
-  arb.io.out.ready := true.B
-
-  (core.io.dmem zip dmemDs).zipWithIndex.foreach { case ((coreMem, tileNode), i) =>
-    coreMem.d.bits := tileNode.bits
-    coreMem.d.valid := tileNode.valid && matchingSources(i)
-    tileNode.ready := coreMem.d.ready && matchingSources(i)
-  }
-
-  (core.io.dmem zip outer.dmemNodes).foreach { case (coreMem, tileNode) =>
-    coreMem.a <> tileNode.out.head._1.a
-  }
-
-  core.io.fpu := DontCare
 
   // TODO eliminate this redundancy
   // val h = dcachePorts.size
@@ -260,6 +273,7 @@ class VortexTileModuleImp(outer: VortexTile) extends BaseTileModuleImp(outer) {
   // dcacheArb.io.requestor <> dcachePorts.toSeq
 }
 
+// FIXME: unsure this is necessary
 trait HasFpuOpt { this: RocketTileModuleImp =>
   val fpuOpt = outer.tileParams.core.fpu.map(params => Module(new FPU(params)(outer.p)))
 }

src/main/scala/tilelink/Coalescing.scala

@@ -59,16 +59,17 @@ case class CoalescerConfig(
   queueDepth: Int,        // request window per lane
   waitTimeout: Int,       // max cycles to wait before forced fifo dequeue, per lane
   addressWidth: Int,      // assume <= 32
-  dataBusWidth: Int,      // memory-side downstream TileLink data bus size.
+  dataBusWidth: Int,      // memory-side downstream TileLink data bus size.  Nominally, this has
+                          // to be the maximum coalLogSizes.
                           // This data bus carries the data bits of coalesced request/responses,
                           // and so it has to be at least wider than word size for the coalescer
-                          // to perform well
+                          // to perform well.
   coalLogSizes: Seq[Int], // list of coalescer sizes to try in the MonoCoalescers
                           // each size is log(byteSize)
                           // max value should match dataBusWidth as the largest-possible
                           // single-beat coealsced size.
   // watermark = 2,       // minimum buffer occupancy to start coalescing
-  wordSizeInBytes: Int,   // 32-bit system
+  wordSizeInBytes: Int,   // word size of the request that each lane makes
   numOldSrcIds: Int,      // num of outstanding requests per lane, from processor
   numNewSrcIds: Int,      // num of outstanding coalesced requests
   respQueueDepth: Int,    // depth of the response fifo queues
@@ -119,19 +120,19 @@ object defaultConfig extends CoalescerConfig(
 )
 
 class CoalescingUnit(config: CoalescerConfig)(implicit p: Parameters) extends LazyModule {
+  // WIP:
   // Nexus node that captures the incoming TL requests, rewrites coalescable requests,
   // and arbitrates between non-coalesced and coalesced requests to a fix number of outputs
   // before sending it out to memory. This node is what's visible to upstream and downstream nodes.
-
+  //
-  // WIP:
+  //  val node = TLNexusNode(
-//  val node = TLNexusNode(
+  //    clientFn  = c => c.head,
-//    clientFn  = c => c.head,
+  //    managerFn = m => m.head  // assuming arbiter generated ids are distinct between edges
-//    managerFn = m => m.head  // assuming arbiter generated ids are distinct between edges
+  //  )
-//  )
+  //  node.in.map(_._2).foreach(edge => require(edge.manager.beatBytes == config.wordSizeInBytes,
-//  node.in.map(_._2).foreach(edge => require(edge.manager.beatBytes == config.wordSizeInBytes,
+  //    s"input edges into coalescer node does not have beatBytes = ${config.wordSizeInBytes}"))
-//    s"input edges into coalescer node does not have beatBytes = ${config.wordSizeInBytes}"))
+  //  node.out.map(_._2).foreach(edge => require(edge.manager.beatBytes == config.maxCoalLogSize,
-//  node.out.map(_._2).foreach(edge => require(edge.manager.beatBytes == config.maxCoalLogSize,
+  //    s"output edges into coalescer node does not have beatBytes = ${config.maxCoalLogSize}"))
-//    s"output edges into coalescer node does not have beatBytes = ${config.maxCoalLogSize}"))
 
   val aggregateNode = TLIdentityNode()
   val cpuNode = TLIdentityNode()
@@ -151,7 +152,12 @@ class CoalescingUnit(config: CoalescerConfig)(implicit p: Parameters) extends La
     Seq(TLMasterPortParameters.v1(coalParam))
   )
 
-  // merge coalescerNode and cpuNode
+  // Merge coalescerNode and cpuNode
+  //
+  // Expand per-lane requests to the wide coalesced size.  As a result, the
+  // output edges of the coalescer all have the same wide width.  This
+  // simplifies cache interface where the cache can always serve same-sized
+  // wide requests, and the coalescer handles taking the right bytes.
   aggregateNode :=* coalescerNode
   aggregateNode :=* TLWidthWidget(config.wordSizeInBytes) :=* cpuNode
 
@@ -1447,11 +1453,11 @@ class MemTraceDriverImp(
     val subword = req.size < log2Ceil(config.wordSizeInBytes).U
 
     // `mask` is currently unused
-    val mask = Wire(UInt(config.wordSizeInBytes.W))
+    // val mask = Wire(UInt(config.wordSizeInBytes.W))
     val wordData = Wire(UInt((config.wordSizeInBytes * 8 * 2).W))
     val sizeInBytes = Wire(UInt((sizeW + 1).W))
     sizeInBytes := (1.U) << req.size
-    mask := Mux(subword, (~((~0.U(64.W)) << sizeInBytes)) << offsetInWord, ~0.U)
+    // mask := Mux(subword, (~((~0.U(64.W)) << sizeInBytes)) << offsetInWord, ~0.U)
     wordData := Mux(subword, req.data << (offsetInWord * 8.U), req.data)
     val wordAlignedAddress =
       req.address & ~((1 << log2Ceil(config.wordSizeInBytes)) - 1).U(addrW.W)
@@ -1697,7 +1703,8 @@ class MemTraceLogger(
         }
         val trailingZerosInMask = trailingZeros(tlIn.a.bits.mask)
         val dataW = tlIn.params.dataBits
-        val mask = ~(~(0.U(dataW.W)) << ((1.U << tlIn.a.bits.size) * 8.U))
+        val sizeInBits = (1.U(1.W) << tlIn.a.bits.size) << 3.U
+        val mask = ~(~(0.U(dataW.W)) << sizeInBits)
         req.data := mask & (tlIn.a.bits.data >> (trailingZerosInMask * 8.U))
         // when (req.valid) {
         //   printf("trailingZerosInMask=%d, mask=%x, data=%x\n", trailingZerosInMask, mask, req.data)