From af01e39b5a9138db5dc2142de6e22358a9d99e94 Mon Sep 17 00:00:00 2001 From: Hansung Kim Date: Sat, 27 May 2023 13:59:25 -0700 Subject: [PATCH] Revamp dataflow between uncoalescer and inflight table Now uncoalescer no longer handles constructing and enqueueing entries to the inflight table; it simply queries the table and works with the found row. Enqueueing / looking-up of the table is clearly split between request and response flow. TODO: delegate sourceId gen to inflight table and remove CoalSourceGen --- src/main/scala/tilelink/Coalescing.scala | 191 ++++++++++++----------- 1 file changed, 99 insertions(+), 92 deletions(-) diff --git a/src/main/scala/tilelink/Coalescing.scala b/src/main/scala/tilelink/Coalescing.scala index 8faa4c8..f0ac6a6 100644 --- a/src/main/scala/tilelink/Coalescing.scala +++ b/src/main/scala/tilelink/Coalescing.scala @@ -790,7 +790,8 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig) reqQueues.io.coalescable := coalescer.io.coalescable reqQueues.io.invalidate := coalescer.io.invalidate - val uncoalescer = Module(new Uncoalescer(config, nonCoalReqT, coalReqT)) + val inflightTable = Module(new InFlightTable(config, nonCoalReqT, coalReqT)) + val uncoalescer = Module(new Uncoalescer(config, inflightTable.entryT)) // =========================================================================== // Request flow @@ -871,14 +872,22 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig) val coalSourceGen = Module(new CoalescerSourceGen(config, coalReqT, tlCoal.d.bits)) coalSourceGen.io.inReq <> coalescer.io.coalReq coalSourceGen.io.inResp <> tlCoal.d - // downstream backpressure on the coalesced edge - coalSourceGen.io.outReq.ready := tlCoal.a.ready - // This is the final coalesced request. - val coalReq = coalSourceGen.io.outReq - dontTouch(coalReq) + // InflightTable IO + // + // Connect coalesced request to be recorded in the uncoalescer table. + inflightTable.io.inCoalReq <> coalSourceGen.io.outReq + inflightTable.io.invalidate := coalescer.io.invalidate + inflightTable.io.windowElts := reqQueues.io.elts + + // This is the final coalesced request. + val coalReq = inflightTable.io.outCoalReq + // downstream backpressure on the coalesced edge + // TODO: custom <>? + inflightTable.io.outCoalReq.ready := tlCoal.a.ready tlCoal.a.valid := coalReq.valid tlCoal.a.bits := coalReq.bits.toTLA(edgeCoal) + dontTouch(coalReq) tlCoal.b.ready := true.B tlCoal.c.valid := false.B @@ -984,22 +993,17 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig) dontTouch(tlOut.d) } - // Uncoalescer input + // Uncoalescer IO // - // connect coalesced request to be recorded in the uncoalescer table - // Only record to inflight table when fire; otherwise a_valid might be up for - // multiple cycles waiting for a_ready, writing bogus data to the row - // FIXME: awkward; make uncoalescer.io.coalReq decoupled - uncoalescer.io.coalReq.valid := coalReq.fire - uncoalescer.io.coalReq.bits := coalReq.bits - uncoalescer.io.invalidate := coalescer.io.invalidate - uncoalescer.io.windowElts := reqQueues.io.elts - // coalesced response to be used to look up the uncoalescer table + // Connect coalesced response uncoalescer.io.coalResp.valid := tlCoal.d.valid uncoalescer.io.coalResp.bits.fromTLD(tlCoal.d.bits) + // Connect lookup result from InflightTable + uncoalescer.io.inflightLookup <> inflightTable.io.lookup + // InflightTable IO: Look up the table with incoming coalesced responses + // FIXME: this should be done inside uncoalescer + inflightTable.io.lookupSourceId := tlCoal.d.bits.source - // Uncoalescer output - // // Connect uncoalescer results back into response queue (respQueues zip uncoalescer.io.respQueueIO).foreach { case (q, uncoalEnqs) => require(q.io.enq.length == config.queueDepth + respQueueUncoalPortOffset, @@ -1026,80 +1030,19 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig) class Uncoalescer( config: CoalescerConfig, - nonCoalReqT: NonCoalescedRequest, - coalReqT: CoalescedRequest, + inflightEntryT: InFlightTableEntry ) extends Module { - val inflightTable = Module(new InFlightTable(config)) val io = IO(new Bundle { - // generated coalesced request, connected to the output of the coalescer. - // val coalReq = Flipped(DecoupledIO(coalReqT.cloneType)) - val coalReq = Input(Valid(coalReqT.cloneType)) - // invalidate signal coming out of coalescer. - val invalidate = Input(Valid(Vec(config.numLanes, UInt(config.queueDepth.W)))) - // coalescing window, connected to the contents of the request queues. - // Uncoalescer looks at the queue entries that got coalesced into `coalReq` - // in order to record which lanes this coalReq originally came from. - // We only care about window.elts because the coalescer would have made - // sure it only looked at the valid entries. - // TODO: duplicate type construction - val windowElts = Input(Vec(config.numLanes, Vec(config.queueDepth, nonCoalReqT))) + val inflightLookup = Flipped(Decoupled(inflightEntryT)) val coalResp = Flipped(Decoupled(new CoalescedResponse(config))) val respQueueIO = Vec(config.numLanes, Vec(config.queueDepth, Decoupled(new NonCoalescedResponse(config))) ) }) - // Inflight table being full is equivalent to source ID being exhausted. - // Therefore, it should never be possible for inflight table to be full and - // coalescer to be firing a valid request at the same time. - // TODO: inflight table is really a more sophisticated sourcegen. Let it - // also take care of sourcegen instead of having a separte pass - // (CoalescerSourceGen). - // when(!inflightTable.io.enq.ready) { - // assert(!io.coalReq.valid, - // "tried to fire a coalesced request when uncoalescer is not ready") - // } - - // Construct a new entry for the inflight table using generated coalesced request - def generateInflightTableEntry: InFlightTableEntry = { - val newEntry = Wire(inflightTable.entryT) - newEntry.source := io.coalReq.bits.source - // Do a 2-D copy from every (numLanes * queueDepth) invalidate output of the - // coalescer to every (numLanes * queueDepth) entry in the inflight table. - (newEntry.lanes zip io.invalidate.bits).zipWithIndex - .foreach { case ((laneEntry, laneInv), lane) => - (laneEntry.reqs zip laneInv.asBools).zipWithIndex - .foreach { case ((reqEntry, inv), i) => - val req = io.windowElts(lane)(i) - when((io.invalidate.valid && inv)) { - printf( - s"coalescer: reqQueue($lane)($i) got invalidated (source=%d)\n", - req.source - ) - } - reqEntry.valid := (io.invalidate.valid && inv) - reqEntry.source := req.source - reqEntry.offset := ((req.address % (1 << config.maxCoalLogSize).U) >> config.wordSizeWidth) - reqEntry.sizeEnum := config.sizeEnum.logSizeToEnum(req.size) - // TODO: load/store op - } - } - assert( - !((io.coalReq.fire === true.B) && (io.coalResp.fire === true.B) && - (newEntry.source === io.coalResp.bits.source)), - "inflight table: enqueueing and looking up the same srcId at the same cycle is not handled" - ) - dontTouch(newEntry) - - newEntry - } - inflightTable.io.enq.valid := io.coalReq.valid - inflightTable.io.enq.bits := generateInflightTableEntry - - // Look up the table with incoming coalesced responses - inflightTable.io.lookup.ready := io.coalResp.valid - inflightTable.io.lookupSourceId := io.coalResp.bits.source io.coalResp.ready := true.B // FIXME, see sw model implementation + // Lookup table whenever coalesced response is valid + io.inflightLookup.ready := io.coalResp.valid // Un-coalescing logic // @@ -1128,7 +1071,7 @@ class Uncoalescer( // Un-coalesce responses back to individual lanes // Connect uncoalesced results back into each lane's response queue - val foundRow = inflightTable.io.lookup.bits + val foundRow = io.inflightLookup.bits (foundRow.lanes zip io.respQueueIO).zipWithIndex.foreach { case ((foundLane, ioEnqs), lane) => foundLane.reqs.zipWithIndex.foreach { case (foundReq, depth) => val ioEnq = ioEnqs(depth) @@ -1148,7 +1091,7 @@ class Uncoalescer( // } // dontTouch(q.io.enq(respQueueCoalPortOffset)) - when(inflightTable.io.lookup.valid && foundReq.valid) { + when(io.inflightLookup.valid && foundReq.valid) { ioEnq.valid := io.coalResp.valid && foundReq.valid ioEnq.bits.source := foundReq.source val logSize = foundRow.sizeEnumT.enumToLogSize(foundReq.sizeEnum) @@ -1166,11 +1109,15 @@ class Uncoalescer( } // InflightCoalReqTable is a table structure that records -// for each unanswered coalesced request which lane the request originated +// for each unanswered coalesced request which lanes the request originated // from, what their original TileLink sourceId were, etc. We use this info to // split the coalesced response back to individual per-lane responses with the // right metadata. -class InFlightTable(config: CoalescerConfig) extends Module { +class InFlightTable( + config: CoalescerConfig, + nonCoalReqT: NonCoalescedRequest, + coalReqT: CoalescedRequest, +) extends Module { val offsetBits = config.maxCoalLogSize - config.wordSizeWidth // assumes word offset val entryT = new InFlightTableEntry( config.numLanes, @@ -1189,7 +1136,26 @@ class InFlightTable(config: CoalescerConfig) extends Module { println(s"=========== table sizeEnumBits: ${entryT.sizeEnumT.getWidth}") val io = IO(new Bundle { - val enq = Flipped(Decoupled(entryT)) + // Enqueue IO + // + // generated coalesced request, connected to the output of the coalescer. + // val coalReq = Flipped(DecoupledIO(coalReqT.cloneType)) + // Valid instead of Flipped(Decoupled), because we have to worry about setting + // the ready bit right, which is better done from CoalSourceGen. + val inCoalReq = Flipped(Decoupled(coalReqT)) + // invalidate signal coming out of coalescer. Needed to generate new entry + // for the table. + val invalidate = Input(Valid(Vec(config.numLanes, UInt(config.queueDepth.W)))) + // coalescing window, connected to the contents of the request queues. + // Need this to generate new entry for the table. + // TODO: duplicate type construction + val windowElts = Input(Vec(config.numLanes, Vec(config.queueDepth, nonCoalReqT))) + // InflightTable also handles sourceID allocation. This is the final + // coalesced request that goes to TileLink with a valid sourceId attached. + val outCoalReq = Decoupled(coalReqT) + + // Lookup outputs + // // TODO: return actual stuff val lookup = Decoupled(entryT) // TODO: put this inside decoupledIO @@ -1223,17 +1189,53 @@ class InFlightTable(config: CoalescerConfig) extends Module { dontTouch(full) // Enqueue logic - io.enq.ready := !full - val enqFire = io.enq.ready && io.enq.valid + // + // Construct a new entry for the inflight table using the coalesced request + def generateInflightTableEntry: InFlightTableEntry = { + val newEntry = Wire(entryT) + newEntry.source := io.inCoalReq.bits.source + // Do a 2-D copy from every (numLanes * queueDepth) invalidate output of the + // coalescer to every (numLanes * queueDepth) entry in the inflight table. + (newEntry.lanes zip io.invalidate.bits).zipWithIndex + .foreach { case ((laneEntry, laneInv), lane) => + (laneEntry.reqs zip laneInv.asBools).zipWithIndex + .foreach { case ((reqEntry, inv), i) => + val req = io.windowElts(lane)(i) + when((io.invalidate.valid && inv)) { + printf( + s"coalescer: reqQueue($lane)($i) got invalidated (source=%d)\n", + req.source + ) + } + reqEntry.valid := (io.invalidate.valid && inv) + reqEntry.source := req.source + reqEntry.offset := ((req.address % (1 << config.maxCoalLogSize).U) >> config.wordSizeWidth) + reqEntry.sizeEnum := config.sizeEnum.logSizeToEnum(req.size) + // TODO: load/store op + } + } + dontTouch(newEntry) + + newEntry + } + + io.outCoalReq <> io.inCoalReq // FIXME: do sourceId allocation + + val enqReady = !full + // Make sure to respect downstream ready here as well; otherwise inCoalReq.valid + // might be up for multiple cycles waiting for outCoalReq.ready, writing bogus + // data to the row + val enqFire = enqReady && io.inCoalReq.valid && io.outCoalReq.ready + val enqSource = io.inCoalReq.bits.source when(enqFire) { // TODO: handle enqueueing and looking up the same entry in the same cycle? - val entryToWrite = table(io.enq.bits.source) + val entryToWrite = table(enqSource) assert( !entryToWrite.valid, "tried to enqueue to an already occupied entry" ) entryToWrite.valid := true.B - entryToWrite.bits := io.enq.bits + entryToWrite.bits := generateInflightTableEntry } // Lookup logic @@ -1249,6 +1251,11 @@ class InFlightTable(config: CoalescerConfig) extends Module { when(io.lookup.fire) { table(io.lookupSourceId).valid := false.B } + assert( + !((enqFire === true.B) && (io.lookup.fire === true.B) && + (enqSource === io.lookupSourceId)), + "inflight table: enqueueing and looking up the same srcId at the same cycle is not handled" + ) dontTouch(io.lookup) }