This commit is contained in:
Hansung Kim
2024-01-19 15:16:37 -08:00
parent fec3d61dd6
commit 737f24fd77

View File

@@ -14,12 +14,17 @@ import freechips.rocketchip.tilelink._
// Note: numNewSrcId is not a part of CoreParam, because the SIMT core should be agnostic to how inflight coalesced request can be genertated // Note: numNewSrcId is not a part of CoreParam, because the SIMT core should be agnostic to how inflight coalesced request can be genertated
case class SIMTCoreParams(nLanes: Int = 4, nSrcIds: Int = 8) case class SIMTCoreParams(nLanes: Int = 4, nSrcIds: Int = 8)
case class MemtraceCoreParams(tracefilename: String = "undefined", traceHasSource: Boolean = false) case class MemtraceCoreParams(
tracefilename: String = "undefined",
traceHasSource: Boolean = false
)
case class CoalXbarParam() case class CoalXbarParam()
case object SIMTCoreKey extends Field[Option[SIMTCoreParams]](None /*default*/ ) case object SIMTCoreKey extends Field[Option[SIMTCoreParams]](None /*default*/ )
case object MemtraceCoreKey extends Field[Option[MemtraceCoreParams]](None /*default*/) case object MemtraceCoreKey
case object CoalescerKey extends Field[Option[CoalescerConfig]](None /*default*/) extends Field[Option[MemtraceCoreParams]](None /*default*/ )
case object CoalescerKey
extends Field[Option[CoalescerConfig]](None /*default*/ )
case object CoalXbarKey extends Field[Option[CoalXbarParam]](None /*default*/ ) case object CoalXbarKey extends Field[Option[CoalXbarParam]](None /*default*/ )
trait InFlightTableSizeEnum extends ChiselEnum { trait InFlightTableSizeEnum extends ChiselEnum {
@@ -81,19 +86,25 @@ case class CoalescerConfig(
) { ) {
// maximum coalesced size // maximum coalesced size
def maxCoalLogSize: Int = { def maxCoalLogSize: Int = {
require(coalLogSizes.max <= dataBusWidth, require(
"multi-beat coalesced reads/writes are currently not supported") coalLogSizes.max <= dataBusWidth,
"multi-beat coalesced reads/writes are currently not supported"
)
if (coalLogSizes.max < dataBusWidth) { if (coalLogSizes.max < dataBusWidth) {
println("======== Warning: coalescer's max coalescing size is set to " + println(
"======== Warning: coalescer's max coalescing size is set to " +
s"${coalLogSizes.max}, which is narrower than data bus width " + s"${coalLogSizes.max}, which is narrower than data bus width " +
s"${dataBusWidth}. This might indicate misconfiguration.") s"${dataBusWidth}. This might indicate misconfiguration."
)
} }
coalLogSizes.max coalLogSizes.max
} }
def wordSizeWidth: Int = { def wordSizeWidth: Int = {
val w = log2Ceil(wordSizeInBytes) val w = log2Ceil(wordSizeInBytes)
require(wordSizeInBytes == 1 << w, require(
s"wordSizeInBytes (${wordSizeInBytes}) is not power of two") wordSizeInBytes == 1 << w,
s"wordSizeInBytes (${wordSizeInBytes}) is not power of two"
)
w w
} }
} }
@@ -167,9 +178,16 @@ class CoalescingUnit(config: CoalescerConfig)(implicit p: Parameters) extends La
// Protocol-agnostic bundles that represent a request and a response to the // Protocol-agnostic bundles that represent a request and a response to the
// coalescer. // coalescer.
class Request(sourceWidth: Int, sizeWidth: Int, addressWidth: Int, dataWidth: Int) class Request(
extends Bundle { sourceWidth: Int,
require(dataWidth % 8 == 0, s"dataWidth (${dataWidth} bits) is not multiple of 8") sizeWidth: Int,
addressWidth: Int,
dataWidth: Int
) extends Bundle {
require(
dataWidth % 8 == 0,
s"dataWidth (${dataWidth} bits) is not multiple of 8"
)
val op = Bool() // 0=READ 1=WRITE val op = Bool() // 0=READ 1=WRITE
val address = UInt(addressWidth.W) val address = UInt(addressWidth.W)
val size = UInt(sizeWidth.W) val size = UInt(sizeWidth.W)
@@ -213,7 +231,10 @@ case class CoalescedRequest(config: CoalescerConfig)
class Response(sourceWidth: Int, sizeWidth: Int, dataWidth: Int) class Response(sourceWidth: Int, sizeWidth: Int, dataWidth: Int)
extends Bundle { extends Bundle {
require(dataWidth % 8 == 0, s"dataWidth (${dataWidth} bits) is not multiple of 8") require(
dataWidth % 8 == 0,
s"dataWidth (${dataWidth} bits) is not multiple of 8"
)
val op = UInt(1.W) // 0=READ 1=WRITE val op = UInt(1.W) // 0=READ 1=WRITE
val size = UInt(sizeWidth.W) val size = UInt(sizeWidth.W)
val source = UInt(sourceWidth.W) val source = UInt(sourceWidth.W)
@@ -627,7 +648,7 @@ class MonoCoalescer(
class MultiCoalescer( class MultiCoalescer(
config: CoalescerConfig, config: CoalescerConfig,
queueT: CoalShiftQueue[NonCoalescedRequest], queueT: CoalShiftQueue[NonCoalescedRequest],
coalReqT: CoalescedRequest, coalReqT: CoalescedRequest
) extends Module { ) extends Module {
val invalidateT = Valid(Vec(config.numLanes, UInt(config.reqQueueDepth.W))) val invalidateT = Valid(Vec(config.numLanes, UInt(config.reqQueueDepth.W)))
val io = IO(new Bundle { val io = IO(new Bundle {
@@ -864,7 +885,9 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
reqQueues.io.coalescable := coalescer.io.coalescable reqQueues.io.coalescable := coalescer.io.coalescable
reqQueues.io.invalidate := coalescer.io.invalidate reqQueues.io.invalidate := coalescer.io.invalidate
val inflightTable = Module(new InFlightTable(config, nonCoalReqT, coalReqT, coalRespT)) val inflightTable = Module(
new InFlightTable(config, nonCoalReqT, coalReqT, coalRespT)
)
val uncoalescer = Module(new Uncoalescer(config, inflightTable.entryT)) val uncoalescer = Module(new Uncoalescer(config, inflightTable.entryT))
// =========================================================================== // ===========================================================================
@@ -942,7 +965,9 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
// │ RespQueues ├─┤ Uncoalescer ├─┤ CoalSourceGen(reclaim) ├─┤ InFlightTable ├── TileLink resp // │ RespQueues ├─┤ Uncoalescer ├─┤ CoalSourceGen(reclaim) ├─┤ InFlightTable ├── TileLink resp
// └────────────┘ └─────────────┘ └────────────────────────┘ └───────────────┘ // └────────────┘ └─────────────┘ └────────────────────────┘ └───────────────┘
// //
val coalSourceGen = Module(new CoalescerSourceGen(config, coalReqT, tlCoal.d.bits)) val coalSourceGen = Module(
new CoalescerSourceGen(config, coalReqT, tlCoal.d.bits)
)
coalSourceGen.io.inReq <> coalescer.io.coalReq coalSourceGen.io.inReq <> coalescer.io.coalReq
// InflightTable IO // InflightTable IO
@@ -1043,8 +1068,10 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
// MultiPortQueue, and eventually serialized. // MultiPortQueue, and eventually serialized.
respQueue.io.enq(respQueueNoncoalPort).valid := tlOut.d.valid respQueue.io.enq(respQueueNoncoalPort).valid := tlOut.d.valid
respQueue.io.enq(respQueueNoncoalPort).bits := resp respQueue.io.enq(respQueueNoncoalPort).bits := resp
assert(respQueue.io.deq.length == 1, assert(
"respQueue should have only one dequeue port to the upstream") respQueue.io.deq.length == 1,
"respQueue should have only one dequeue port to the upstream"
)
respQueue.io.deq.head.ready := tlIn.d.ready respQueue.io.deq.head.ready := tlIn.d.ready
tlIn.d.valid := respQueue.io.deq.head.valid tlIn.d.valid := respQueue.io.deq.head.valid
@@ -1074,7 +1101,8 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
// //
// Connect coalesced response // Connect coalesced response
uncoalescer.io.coalResp.valid := coalSourceGen.io.inResp.valid uncoalescer.io.coalResp.valid := coalSourceGen.io.inResp.valid
uncoalescer.io.coalResp.bits.fromTLD(coalSourceGen.io.inResp.bits, coalSourceGen.io.inResp.fire) uncoalescer.io.coalResp.bits
.fromTLD(coalSourceGen.io.inResp.bits, coalSourceGen.io.inResp.fire)
coalSourceGen.io.inResp.ready := uncoalescer.io.coalResp.ready coalSourceGen.io.inResp.ready := uncoalescer.io.coalResp.ready
// Connect lookup result from InflightTable // Connect lookup result from InflightTable
@@ -1086,15 +1114,18 @@ class CoalescingUnitImp(outer: CoalescingUnit, config: CoalescerConfig)
inflightTable.io.lookupSourceId.bits := coalSourceGen.io.inResp.bits.source inflightTable.io.lookupSourceId.bits := coalSourceGen.io.inResp.bits.source
// Connect uncoalescer results back into response queue // Connect uncoalescer results back into response queue
(respQueues zip uncoalescer.io.respQueueIO).zipWithIndex.foreach (respQueues zip uncoalescer.io.respQueueIO).zipWithIndex.foreach {
{ case ((q, sameLaneUncoalResps), lane) => case ((q, sameLaneUncoalResps), lane) =>
// reqQueueDepth here is the maximum number of same-lane, different-time // reqQueueDepth here is the maximum number of same-lane, different-time
// requests that can go into a single coalesced response. We need to have // requests that can go into a single coalesced response. We need to have
// that many enq ports to not backpressure the uncoalescer. // that many enq ports to not backpressure the uncoalescer.
require(q.io.enq.length == config.reqQueueDepth + respQueueUncoalPortOffset, require(
s"wrong number of enq ports for MultiPort response queue") q.io.enq.length == config.reqQueueDepth + respQueueUncoalPortOffset,
s"wrong number of enq ports for MultiPort response queue"
)
// slice the ports reserved for uncoalesced response // slice the ports reserved for uncoalesced response
val sameLaneEnqPorts = q.io.enq.slice(respQueueUncoalPortOffset, q.io.enq.length) val sameLaneEnqPorts =
q.io.enq.slice(respQueueUncoalPortOffset, q.io.enq.length)
(sameLaneEnqPorts zip sameLaneUncoalResps).foreach { (sameLaneEnqPorts zip sameLaneUncoalResps).foreach {
case (enqPort, uncoalResp) => { case (enqPort, uncoalResp) => {
enqPort <> uncoalResp enqPort <> uncoalResp
@@ -1127,7 +1158,8 @@ class Uncoalescer(
val io = IO(new Bundle { val io = IO(new Bundle {
val inflightLookup = Flipped(Decoupled(inflightEntryT)) val inflightLookup = Flipped(Decoupled(inflightEntryT))
val coalResp = Flipped(Decoupled(new CoalescedResponse(config))) val coalResp = Flipped(Decoupled(new CoalescedResponse(config)))
val respQueueIO = Vec(config.numLanes, val respQueueIO = Vec(
config.numLanes,
// reqQueueDepth because if we're doing time-coalescing, that's the // reqQueueDepth because if we're doing time-coalescing, that's the
// maximum number of same-lane, different-time requests that can go into // maximum number of same-lane, different-time requests that can go into
// a single coalesced request. // a single coalesced request.
@@ -1137,7 +1169,12 @@ class Uncoalescer(
// Un-coalescing logic // Un-coalescing logic
// //
def getCoalescedDataChunk(data: UInt, dataWidth: Int, offset: UInt, logSize: UInt): UInt = { def getCoalescedDataChunk(
data: UInt,
dataWidth: Int,
offset: UInt,
logSize: UInt
): UInt = {
assert(logSize === 2.U, "currently only supporting 4-byte accesses. TODO") assert(logSize === 2.U, "currently only supporting 4-byte accesses. TODO")
// sizeInBits should be simulation-only construct // sizeInBits should be simulation-only construct
@@ -1167,19 +1204,25 @@ class Uncoalescer(
// ready. This is necessary because uncoalescing logic is a combinational // ready. This is necessary because uncoalescing logic is a combinational
// logic that produces all the split responses at the same cycle, so it needs // logic that produces all the split responses at the same cycle, so it needs
// to be guaranteed that all of them has somewhere to go. // to be guaranteed that all of them has somewhere to go.
val allRespQueueEnqReady = io.respQueueIO.map(_.map(_.ready).reduce(_ && _)).reduce(_ && _) val allRespQueueEnqReady =
io.respQueueIO.map(_.map(_.ready).reduce(_ && _)).reduce(_ && _)
tablePipeRegDeq.ready := allRespQueueEnqReady tablePipeRegDeq.ready := allRespQueueEnqReady
coalRespPipeRegDeq.ready := allRespQueueEnqReady coalRespPipeRegDeq.ready := allRespQueueEnqReady
assert(io.coalResp.fire === io.inflightLookup.fire, assert(
"enqueue timing for uncoalescer pipeline registers out-of-sync!") io.coalResp.fire === io.inflightLookup.fire,
assert(tablePipeRegDeq.fire === coalRespPipeRegDeq.fire, "enqueue timing for uncoalescer pipeline registers out-of-sync!"
"dequeue timing for uncoalescer pipeline registers out-of-sync!") )
assert(
tablePipeRegDeq.fire === coalRespPipeRegDeq.fire,
"dequeue timing for uncoalescer pipeline registers out-of-sync!"
)
// Un-coalesce responses back to individual lanes. Connect uncoalesced // Un-coalesce responses back to individual lanes. Connect uncoalesced
// results back into each lane's response queue. // results back into each lane's response queue.
val tableRow = tablePipeRegDeq val tableRow = tablePipeRegDeq
(io.respQueueIO zip tableRow.bits.lanes).zipWithIndex.foreach { case ((enqIOs, lane), laneNum) => (io.respQueueIO zip tableRow.bits.lanes).zipWithIndex.foreach {
case ((enqIOs, lane), laneNum) =>
lane.reqs.zipWithIndex.foreach { case (req, depth) => lane.reqs.zipWithIndex.foreach { case (req, depth) =>
val enqIO = enqIOs(depth) val enqIO = enqIOs(depth)
enqIO.valid := false.B enqIO.valid := false.B
@@ -1219,9 +1262,10 @@ class InFlightTable(
config: CoalescerConfig, config: CoalescerConfig,
nonCoalReqT: NonCoalescedRequest, nonCoalReqT: NonCoalescedRequest,
coalReqT: CoalescedRequest, coalReqT: CoalescedRequest,
coalRespT: CoalescedResponse, coalRespT: CoalescedResponse
) extends Module { ) extends Module {
val offsetBits = config.maxCoalLogSize - config.wordSizeWidth // assumes word offset val offsetBits =
config.maxCoalLogSize - config.wordSizeWidth // assumes word offset
val entryT = new InFlightTableEntry( val entryT = new InFlightTableEntry(
config.numLanes, config.numLanes,
config.reqQueueDepth, config.reqQueueDepth,
@@ -1243,11 +1287,13 @@ class InFlightTable(
val inCoalReq = Flipped(Decoupled(coalReqT)) val inCoalReq = Flipped(Decoupled(coalReqT))
// invalidate signal coming out of coalescer. Needed to generate new entry // invalidate signal coming out of coalescer. Needed to generate new entry
// for the table. // for the table.
val invalidate = Input(Valid(Vec(config.numLanes, UInt(config.reqQueueDepth.W)))) val invalidate =
Input(Valid(Vec(config.numLanes, UInt(config.reqQueueDepth.W))))
// coalescing window, connected to the contents of the request queues. // coalescing window, connected to the contents of the request queues.
// Need this to generate new entry for the table. // Need this to generate new entry for the table.
// TODO: duplicate type construction // TODO: duplicate type construction
val windowElts = Input(Vec(config.numLanes, Vec(config.reqQueueDepth, nonCoalReqT))) val windowElts =
Input(Vec(config.numLanes, Vec(config.reqQueueDepth, nonCoalReqT)))
// InflightTable simply passes through the inCoalReq to outCoalReq, only snooping // InflightTable simply passes through the inCoalReq to outCoalReq, only snooping
// on its data to record what's necessary. // on its data to record what's necessary.
val outCoalReq = Decoupled(coalReqT) val outCoalReq = Decoupled(coalReqT)
@@ -1347,23 +1393,27 @@ class InFlightTable(
} }
// Lookup logic // Lookup logic
io.lookupResult.valid := io.lookupSourceId.valid && table(io.lookupSourceId.bits).valid io.lookupResult.valid := io.lookupSourceId.valid && table(
io.lookupSourceId.bits
).valid
io.lookupResult.bits := table(io.lookupSourceId.bits).bits io.lookupResult.bits := table(io.lookupSourceId.bits).bits
// every lookup to the table should succeed as the request should have // every lookup to the table should succeed as the request should have
// gotten recorded earlier than the response // gotten recorded earlier than the response
when(io.lookupSourceId.valid) { when(io.lookupSourceId.valid) {
assert(table(io.lookupSourceId.bits).valid === true.B, assert(
"table lookup with a valid sourceId failed") table(io.lookupSourceId.bits).valid === true.B,
"table lookup with a valid sourceId failed"
)
assert(
!(enqFire && io.lookupResult.fire &&
(enqSource === io.lookupSourceId.bits)),
"inflight table: enqueueing and looking up the same srcId at the same cycle is not handled"
)
} }
// Dequeue as soon as lookup succeeds // Dequeue as soon as lookup succeeds
when(io.lookupResult.fire) { when(io.lookupResult.fire) {
table(io.lookupSourceId.bits).valid := false.B table(io.lookupSourceId.bits).valid := false.B
} }
assert(
!((enqFire === true.B) && (io.lookupResult.fire === true.B) &&
(enqSource === io.lookupSourceId.bits)),
"inflight table: enqueueing and looking up the same srcId at the same cycle is not handled"
)
dontTouch(io.lookupResult) dontTouch(io.lookupResult)
} }
@@ -1403,8 +1453,11 @@ object TLUtils {
"unhandled TL A opcode found" "unhandled TL A opcode found"
) )
} }
Mux(opcode === TLMessages.PutFullData || opcode === TLMessages.PutPartialData, Mux(
true.B, false.B) opcode === TLMessages.PutFullData || opcode === TLMessages.PutPartialData,
true.B,
false.B
)
} }
def DOpcodeIsStore(opcode: UInt, checkOpcode: Bool): Bool = { def DOpcodeIsStore(opcode: UInt, checkOpcode: Bool): Bool = {
when(checkOpcode) { when(checkOpcode) {
@@ -1531,18 +1584,21 @@ class MemTraceDriverImp(
Cat(8.U(4.W), addr(27, 0)) Cat(8.U(4.W), addr(27, 0))
} }
val sourceGens = Seq.fill(config.numLanes)(Module( val sourceGens = Seq.fill(config.numLanes)(
Module(
new SourceGenerator( new SourceGenerator(
log2Ceil(config.numOldSrcIds), log2Ceil(config.numOldSrcIds),
ignoreInUse = false ignoreInUse = false
) )
)) )
)
// Advance source ID for all lanes in synchrony // Advance source ID for all lanes in synchrony
val syncedSourceGenValid = sourceGens.map(_.io.id.valid).reduce(_ && _) val syncedSourceGenValid = sourceGens.map(_.io.id.valid).reduce(_ && _)
// Take requests off of the queue and generate TL requests // Take requests off of the queue and generate TL requests
(outer.laneNodes zip reqQueues).zipWithIndex.foreach { case ((node, reqQ), lane) => (outer.laneNodes zip reqQueues).zipWithIndex.foreach {
case ((node, reqQ), lane) =>
val (tlOut, edge) = node.out(0) val (tlOut, edge) = node.out(0)
val req = reqQ.io.deq.bits val req = reqQ.io.deq.bits
@@ -1821,7 +1877,10 @@ class MemTraceLogger(
// and transaction happened. // and transaction happened.
resp.valid := tlOut.d.fire resp.valid := tlOut.d.fire
resp.size := tlOut.d.bits.size resp.size := tlOut.d.bits.size
resp.is_store := TLUtils.DOpcodeIsStore(tlOut.d.bits.opcode, tlOut.d.fire) resp.is_store := TLUtils.DOpcodeIsStore(
tlOut.d.bits.opcode,
tlOut.d.fire
)
resp.source := tlOut.d.bits.source resp.source := tlOut.d.bits.source
// NOTE: TL D channel doesn't carry address nor mask, so there's no easy // NOTE: TL D channel doesn't carry address nor mask, so there's no easy
// way to figure out which bytes the master actually use. Since we // way to figure out which bytes the master actually use. Since we