Revamp dataflow between uncoalescer and inflight table

Now uncoalescer no longer handles constructing and enqueueing entries
to the inflight table; it simply queries the table and works with the
found row.  Enqueueing / looking-up of the table is clearly split
between request and response flow.

TODO: delegate sourceId gen to inflight table and remove CoalSourceGen

src/main/scala/tilelink/Coalescing.scala

@@ -790,7 +790,8 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
   reqQueues.io.coalescable := coalescer.io.coalescable
   reqQueues.io.invalidate := coalescer.io.invalidate
 
-  val uncoalescer = Module(new Uncoalescer(config, nonCoalReqT, coalReqT))
+  val inflightTable = Module(new InFlightTable(config, nonCoalReqT, coalReqT))
+  val uncoalescer = Module(new Uncoalescer(config, inflightTable.entryT))
 
   // ===========================================================================
   // Request flow
@@ -871,14 +872,22 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
   val coalSourceGen = Module(new CoalescerSourceGen(config, coalReqT, tlCoal.d.bits))
   coalSourceGen.io.inReq <> coalescer.io.coalReq
   coalSourceGen.io.inResp <> tlCoal.d
-  // downstream backpressure on the coalesced edge
-  coalSourceGen.io.outReq.ready := tlCoal.a.ready
-  // This is the final coalesced request.
-  val coalReq = coalSourceGen.io.outReq
-  dontTouch(coalReq)
 
+  // InflightTable IO
+  //
+  // Connect coalesced request to be recorded in the uncoalescer table.
+  inflightTable.io.inCoalReq <> coalSourceGen.io.outReq
+  inflightTable.io.invalidate := coalescer.io.invalidate
+  inflightTable.io.windowElts := reqQueues.io.elts
+
+  // This is the final coalesced request.
+  val coalReq = inflightTable.io.outCoalReq
+  // downstream backpressure on the coalesced edge
+  // TODO: custom <>?
+  inflightTable.io.outCoalReq.ready := tlCoal.a.ready
   tlCoal.a.valid := coalReq.valid
   tlCoal.a.bits := coalReq.bits.toTLA(edgeCoal)
+  dontTouch(coalReq)
 
   tlCoal.b.ready := true.B
   tlCoal.c.valid := false.B
@@ -984,22 +993,17 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
       dontTouch(tlOut.d)
   }
 
-  // Uncoalescer input
+  // Uncoalescer IO
   //
-  // connect coalesced request to be recorded in the uncoalescer table
+  // Connect coalesced response
-  // Only record to inflight table when fire; otherwise a_valid might be up for
-  // multiple cycles waiting for a_ready, writing bogus data to the row
-  // FIXME: awkward; make uncoalescer.io.coalReq decoupled
-  uncoalescer.io.coalReq.valid := coalReq.fire
-  uncoalescer.io.coalReq.bits := coalReq.bits
-  uncoalescer.io.invalidate := coalescer.io.invalidate
-  uncoalescer.io.windowElts := reqQueues.io.elts
-  // coalesced response to be used to look up the uncoalescer table
   uncoalescer.io.coalResp.valid := tlCoal.d.valid
   uncoalescer.io.coalResp.bits.fromTLD(tlCoal.d.bits)
+  // Connect lookup result from InflightTable
+  uncoalescer.io.inflightLookup <> inflightTable.io.lookup
+  // InflightTable IO: Look up the table with incoming coalesced responses
+  // FIXME: this should be done inside uncoalescer
+  inflightTable.io.lookupSourceId := tlCoal.d.bits.source
 
-  // Uncoalescer output
-  //
   // Connect uncoalescer results back into response queue
   (respQueues zip uncoalescer.io.respQueueIO).foreach { case (q, uncoalEnqs) =>
     require(q.io.enq.length == config.queueDepth + respQueueUncoalPortOffset,
@@ -1026,80 +1030,19 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
 
 class Uncoalescer(
     config: CoalescerConfig,
-    nonCoalReqT: NonCoalescedRequest,
+    inflightEntryT: InFlightTableEntry
-    coalReqT: CoalescedRequest,
 ) extends Module {
-  val inflightTable = Module(new InFlightTable(config))
   val io = IO(new Bundle {
-    // generated coalesced request, connected to the output of the coalescer.
+    val inflightLookup = Flipped(Decoupled(inflightEntryT))
-    // val coalReq = Flipped(DecoupledIO(coalReqT.cloneType))
-    val coalReq = Input(Valid(coalReqT.cloneType))
-    // invalidate signal coming out of coalescer.
-    val invalidate = Input(Valid(Vec(config.numLanes, UInt(config.queueDepth.W))))
-    // coalescing window, connected to the contents of the request queues.
-    // Uncoalescer looks at the queue entries that got coalesced into `coalReq`
-    // in order to record which lanes this coalReq originally came from.
-    // We only care about window.elts because the coalescer would have made
-    // sure it only looked at the valid entries.
-    // TODO: duplicate type construction
-    val windowElts = Input(Vec(config.numLanes, Vec(config.queueDepth, nonCoalReqT)))
     val coalResp = Flipped(Decoupled(new CoalescedResponse(config)))
     val respQueueIO = Vec(config.numLanes,
       Vec(config.queueDepth, Decoupled(new NonCoalescedResponse(config)))
     )
   })
 
-  // Inflight table being full is equivalent to source ID being exhausted.
-  // Therefore, it should never be possible for inflight table to be full and
-  // coalescer to be firing a valid request at the same time.
-  // TODO: inflight table is really a more sophisticated sourcegen.  Let it
-  // also take care of sourcegen instead of having a separte pass
-  // (CoalescerSourceGen).
-  // when(!inflightTable.io.enq.ready) {
-  //   assert(!io.coalReq.valid,
-  //     "tried to fire a coalesced request when uncoalescer is not ready")
-  // }
-
-  // Construct a new entry for the inflight table using generated coalesced request
-  def generateInflightTableEntry: InFlightTableEntry = {
-    val newEntry = Wire(inflightTable.entryT)
-    newEntry.source := io.coalReq.bits.source
-    // Do a 2-D copy from every (numLanes * queueDepth) invalidate output of the
-    // coalescer to every (numLanes * queueDepth) entry in the inflight table.
-    (newEntry.lanes zip io.invalidate.bits).zipWithIndex
-      .foreach { case ((laneEntry, laneInv), lane) =>
-        (laneEntry.reqs zip laneInv.asBools).zipWithIndex
-          .foreach { case ((reqEntry, inv), i) =>
-            val req = io.windowElts(lane)(i)
-            when((io.invalidate.valid && inv)) {
-              printf(
-                s"coalescer: reqQueue($lane)($i) got invalidated (source=%d)\n",
-                req.source
-              )
-            }
-            reqEntry.valid := (io.invalidate.valid && inv)
-            reqEntry.source := req.source
-            reqEntry.offset := ((req.address % (1 << config.maxCoalLogSize).U) >> config.wordSizeWidth)
-            reqEntry.sizeEnum := config.sizeEnum.logSizeToEnum(req.size)
-            // TODO: load/store op
-          }
-      }
-    assert(
-      !((io.coalReq.fire === true.B) && (io.coalResp.fire === true.B) &&
-        (newEntry.source === io.coalResp.bits.source)),
-      "inflight table: enqueueing and looking up the same srcId at the same cycle is not handled"
-    )
-    dontTouch(newEntry)
-
-    newEntry
-  }
-  inflightTable.io.enq.valid := io.coalReq.valid
-  inflightTable.io.enq.bits := generateInflightTableEntry
-
-  // Look up the table with incoming coalesced responses
-  inflightTable.io.lookup.ready := io.coalResp.valid
-  inflightTable.io.lookupSourceId := io.coalResp.bits.source
   io.coalResp.ready := true.B // FIXME, see sw model implementation
+  // Lookup table whenever coalesced response is valid
+  io.inflightLookup.ready := io.coalResp.valid
 
   // Un-coalescing logic
   //
@@ -1128,7 +1071,7 @@ class Uncoalescer(
 
   // Un-coalesce responses back to individual lanes
   // Connect uncoalesced results back into each lane's response queue
-  val foundRow = inflightTable.io.lookup.bits
+  val foundRow = io.inflightLookup.bits
   (foundRow.lanes zip io.respQueueIO).zipWithIndex.foreach { case ((foundLane, ioEnqs), lane) =>
     foundLane.reqs.zipWithIndex.foreach { case (foundReq, depth) =>
       val ioEnq = ioEnqs(depth)
@@ -1148,7 +1091,7 @@ class Uncoalescer(
       // }
       // dontTouch(q.io.enq(respQueueCoalPortOffset))
 
-      when(inflightTable.io.lookup.valid && foundReq.valid) {
+      when(io.inflightLookup.valid && foundReq.valid) {
         ioEnq.valid := io.coalResp.valid && foundReq.valid
         ioEnq.bits.source := foundReq.source
         val logSize = foundRow.sizeEnumT.enumToLogSize(foundReq.sizeEnum)
@@ -1166,11 +1109,15 @@ class Uncoalescer(
 }
 
 // InflightCoalReqTable is a table structure that records
-// for each unanswered coalesced request which lane the request originated
+// for each unanswered coalesced request which lanes the request originated
 // from, what their original TileLink sourceId were, etc.  We use this info to
 // split the coalesced response back to individual per-lane responses with the
 // right metadata.
-class InFlightTable(config: CoalescerConfig) extends Module {
+class InFlightTable(
+    config: CoalescerConfig,
+    nonCoalReqT: NonCoalescedRequest,
+    coalReqT: CoalescedRequest,
+) extends Module {
   val offsetBits = config.maxCoalLogSize - config.wordSizeWidth // assumes word offset
   val entryT = new InFlightTableEntry(
     config.numLanes,
@@ -1189,7 +1136,26 @@ class InFlightTable(config: CoalescerConfig) extends Module {
   println(s"=========== table sizeEnumBits: ${entryT.sizeEnumT.getWidth}")
 
   val io = IO(new Bundle {
-    val enq = Flipped(Decoupled(entryT))
+    // Enqueue IO
+    //
+    // generated coalesced request, connected to the output of the coalescer.
+    // val coalReq = Flipped(DecoupledIO(coalReqT.cloneType))
+    // Valid instead of Flipped(Decoupled), because we have to worry about setting
+    // the ready bit right, which is better done from CoalSourceGen.
+    val inCoalReq = Flipped(Decoupled(coalReqT))
+    // invalidate signal coming out of coalescer.  Needed to generate new entry
+    // for the table.
+    val invalidate = Input(Valid(Vec(config.numLanes, UInt(config.queueDepth.W))))
+    // coalescing window, connected to the contents of the request queues.
+    // Need this to generate new entry for the table.
+    // TODO: duplicate type construction
+    val windowElts = Input(Vec(config.numLanes, Vec(config.queueDepth, nonCoalReqT)))
+    // InflightTable also handles sourceID allocation. This is the final
+    // coalesced request that goes to TileLink with a valid sourceId attached.
+    val outCoalReq = Decoupled(coalReqT)
+
+    // Lookup outputs
+    //
     // TODO: return actual stuff
     val lookup = Decoupled(entryT)
     // TODO: put this inside decoupledIO
@@ -1223,17 +1189,53 @@ class InFlightTable(config: CoalescerConfig) extends Module {
   dontTouch(full)
 
   // Enqueue logic
-  io.enq.ready := !full
+  //
-  val enqFire = io.enq.ready && io.enq.valid
+  // Construct a new entry for the inflight table using the coalesced request
+  def generateInflightTableEntry: InFlightTableEntry = {
+    val newEntry = Wire(entryT)
+    newEntry.source := io.inCoalReq.bits.source
+    // Do a 2-D copy from every (numLanes * queueDepth) invalidate output of the
+    // coalescer to every (numLanes * queueDepth) entry in the inflight table.
+    (newEntry.lanes zip io.invalidate.bits).zipWithIndex
+      .foreach { case ((laneEntry, laneInv), lane) =>
+        (laneEntry.reqs zip laneInv.asBools).zipWithIndex
+          .foreach { case ((reqEntry, inv), i) =>
+            val req = io.windowElts(lane)(i)
+            when((io.invalidate.valid && inv)) {
+              printf(
+                s"coalescer: reqQueue($lane)($i) got invalidated (source=%d)\n",
+                req.source
+              )
+            }
+            reqEntry.valid := (io.invalidate.valid && inv)
+            reqEntry.source := req.source
+            reqEntry.offset := ((req.address % (1 << config.maxCoalLogSize).U) >> config.wordSizeWidth)
+            reqEntry.sizeEnum := config.sizeEnum.logSizeToEnum(req.size)
+            // TODO: load/store op
+          }
+      }
+    dontTouch(newEntry)
+
+    newEntry
+  }
+
+  io.outCoalReq <> io.inCoalReq // FIXME: do sourceId allocation
+
+  val enqReady = !full
+  // Make sure to respect downstream ready here as well; otherwise inCoalReq.valid
+  // might be up for multiple cycles waiting for outCoalReq.ready, writing bogus
+  // data to the row
+  val enqFire = enqReady && io.inCoalReq.valid && io.outCoalReq.ready
+  val enqSource = io.inCoalReq.bits.source
   when(enqFire) {
     // TODO: handle enqueueing and looking up the same entry in the same cycle?
-    val entryToWrite = table(io.enq.bits.source)
+    val entryToWrite = table(enqSource)
     assert(
       !entryToWrite.valid,
       "tried to enqueue to an already occupied entry"
     )
     entryToWrite.valid := true.B
-    entryToWrite.bits := io.enq.bits
+    entryToWrite.bits := generateInflightTableEntry
   }
 
   // Lookup logic
@@ -1249,6 +1251,11 @@ class InFlightTable(config: CoalescerConfig) extends Module {
   when(io.lookup.fire) {
     table(io.lookupSourceId).valid := false.B
   }
+  assert(
+    !((enqFire === true.B) && (io.lookup.fire === true.B) &&
+      (enqSource === io.lookupSourceId)),
+    "inflight table: enqueueing and looking up the same srcId at the same cycle is not handled"
+  )
 
   dontTouch(io.lookup)
 }