Merge pull request #1629 from ucb-bar/clusters

Support clustered-cores

.github/scripts/defaults.sh

@@ -33,7 +33,7 @@ grouping["group-peripherals"]="chipyard-dmirocket chipyard-dmiboom chipyard-spif
 grouping["group-accels"]="chipyard-mempress chipyard-sha3 chipyard-hwacha chipyard-gemmini chipyard-manymmioaccels chipyard-nvdla chipyard-aes256ecb"
 grouping["group-constellation"]="chipyard-constellation"
 grouping["group-tracegen"]="tracegen tracegen-boom"
-grouping["group-other"]="icenet testchipip constellation rocketchip-amba rocketchip-tlsimple rocketchip-tlwidth rocketchip-tlxbar"
+grouping["group-other"]="icenet testchipip constellation rocketchip-amba rocketchip-tlsimple rocketchip-tlwidth rocketchip-tlxbar chipyard-clusters"
 grouping["group-fpga"]="arty35t arty100t nexysvideo vc707 vcu118"
 
 # key value store to get the build strings
@@ -67,6 +67,7 @@ mapping["chipyard-shuttle"]=" CONFIG=ShuttleConfig"
 mapping["chipyard-multiclock-rocket"]=" CONFIG=MulticlockRocketConfig"
 mapping["chipyard-nomem-scratchpad"]=" CONFIG=MMIOScratchpadOnlyRocketConfig"
 mapping["chipyard-constellation"]=" CONFIG=SharedNoCConfig"
+mapping["chipyard-clusters"]=" CONFIG=ClusteredRocketConfig verilog"
 mapping["chipyard-aes256ecb"]=" CONFIG=AES256ECBRocketConfig"
 
 mapping["constellation"]=" SUB_PROJECT=constellation"

build.sbt

@@ -95,7 +95,7 @@ lazy val chiselSettings = Seq(
 
 // -- Rocket Chip --
 
-lazy val hardfloat = freshProject("hardfloat", rocketChipDir / "hardfloat/hardfloat")
+lazy val hardfloat = freshProject("hardfloat", file("generators/hardfloat/hardfloat"))
   .settings(chiselSettings)
   .dependsOn(midasTargetUtils)
   .settings(commonSettings)

fpga/src/main/scala/arty/Configs.scala

@@ -30,6 +30,7 @@ class WithArtyTweaks extends Config(
   new chipyard.config.WithFrontBusFrequency(32) ++
   new chipyard.config.WithControlBusFrequency(32) ++
   new chipyard.config.WithPeripheryBusFrequency(32) ++
+  new chipyard.config.WithControlBusFrequency(32) ++
   new testchipip.serdes.WithNoSerialTL ++
   new testchipip.soc.WithNoScratchpads
 )

fpga/src/main/scala/arty100t/Configs.scala

@@ -18,7 +18,7 @@ import chipyard.{BuildSystem}
 
 // don't use FPGAShell's DesignKey
 class WithNoDesignKey extends Config((site, here, up) => {
-  case DesignKey => (p: Parameters) => new SimpleLazyModule()(p)
+  case DesignKey => (p: Parameters) => new SimpleLazyRawModule()(p)
 })
 
 // By default, this uses the on-board USB-UART for the TSI-over-UART link

fpga/src/main/scala/arty100t/Harness.scala

@@ -5,12 +5,12 @@ import chisel3.util._
 import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters}
+import freechips.rocketchip.prci._
 import freechips.rocketchip.subsystem.{SystemBusKey}
 
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks._
 import sifive.fpgashells.ip.xilinx.{IBUF, PowerOnResetFPGAOnly}
 
 import sifive.blocks.devices.uart._

fpga/src/main/scala/nexysvideo/Configs.scala

@@ -18,7 +18,7 @@ import chipyard.{BuildSystem}
 
 // don't use FPGAShell's DesignKey
 class WithNoDesignKey extends Config((site, here, up) => {
-  case DesignKey => (p: Parameters) => new SimpleLazyModule()(p)
+  case DesignKey => (p: Parameters) => new SimpleLazyRawModule()(p)
 })
 
 // DOC include start: WithNexysVideoTweaks and Rocket
@@ -60,6 +60,7 @@ class WithTinyNexysVideoTweaks extends Config(
   new chipyard.config.WithFrontBusFrequency(50.0) ++
   new chipyard.config.WithSystemBusFrequency(50.0) ++
   new chipyard.config.WithPeripheryBusFrequency(50.0) ++
+  new chipyard.config.WithControlBusFrequency(50.0) ++
   new chipyard.harness.WithAllClocksFromHarnessClockInstantiator ++
   new chipyard.clocking.WithPassthroughClockGenerator ++
   new chipyard.config.WithNoDebug ++ // no jtag

fpga/src/main/scala/nexysvideo/Harness.scala

@@ -7,10 +7,10 @@ import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.subsystem.{SystemBusKey}
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks._
 
 import sifive.blocks.devices.uart._
 

fpga/src/main/scala/vc707/TestHarness.scala

@@ -7,11 +7,12 @@ import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.subsystem.{SystemBusKey}
 import freechips.rocketchip.diplomacy.{IdRange, TransferSizes}
+import freechips.rocketchip.prci._
 
 import sifive.fpgashells.shell.xilinx.{VC707Shell, UARTVC707ShellPlacer, PCIeVC707ShellPlacer, ChipLinkVC707PlacedOverlay}
 import sifive.fpgashells.ip.xilinx.{IBUF, PowerOnResetFPGAOnly}
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks.{PLLFactoryKey}
 import sifive.fpgashells.devices.xilinx.xilinxvc707pciex1.{XilinxVC707PCIeX1IO}
 
 import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTPortIO}
@@ -87,6 +88,7 @@ class VC707FPGATestHarness(override implicit val p: Parameters) extends VC707She
 }
 
 class VC707FPGATestHarnessImp(_outer: VC707FPGATestHarness) extends LazyRawModuleImp(_outer) with HasHarnessInstantiators {
+  override def provideImplicitClockToLazyChildren = true
   val vc707Outer = _outer
 
   val reset = IO(Input(Bool())).suggestName("reset")

fpga/src/main/scala/vcu118/Configs.scala

@@ -48,6 +48,7 @@ class WithVCU118Tweaks extends Config(
   new chipyard.config.WithSystemBusFrequency(100) ++
   new chipyard.config.WithControlBusFrequency(100) ++
   new chipyard.config.WithPeripheryBusFrequency(100) ++
+  new chipyard.config.WithControlBusFrequency(100) ++
   new WithFPGAFrequency(100) ++ // default 100MHz freq
   // harness binders
   new WithUART ++

fpga/src/main/scala/vcu118/CustomOverlays.scala

@@ -5,7 +5,7 @@ import chisel3._
 import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.{Parameters, Field}
 import freechips.rocketchip.tilelink.{TLInwardNode, TLAsyncCrossingSink}
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell._
 import sifive.fpgashells.ip.xilinx._
 import sifive.fpgashells.shell.xilinx._
@@ -79,7 +79,7 @@ class DDR2VCU118PlacedOverlay(val shell: VCU118FPGATestHarness, name: String, va
     ui.reset := /*!port.mmcm_locked ||*/ port.c0_ddr4_ui_clk_sync_rst
     port.c0_sys_clk_i := sys.clock.asUInt
     port.sys_rst := sys.reset // pllReset
-    port.c0_ddr4_aresetn := !ar.reset
+    port.c0_ddr4_aresetn := !(ar.reset.asBool)
 
     // This was just copied from the SiFive example, but it's hard to follow.
     // The pins are emitted in the following order:

fpga/src/main/scala/vcu118/TestHarness.scala

@@ -8,11 +8,11 @@ import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.diplomacy.{IdRange, TransferSizes}
 import freechips.rocketchip.subsystem.{SystemBusKey}
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.ip.xilinx.{IBUF, PowerOnResetFPGAOnly}
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks._
 
 import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTPortIO}
 import sifive.blocks.devices.spi.{PeripherySPIKey, SPIPortIO}
@@ -90,6 +90,7 @@ class VCU118FPGATestHarness(override implicit val p: Parameters) extends VCU118S
 }
 
 class VCU118FPGATestHarnessImp(_outer: VCU118FPGATestHarness) extends LazyRawModuleImp(_outer) with HasHarnessInstantiators {
+  override def provideImplicitClockToLazyChildren = true
   val vcu118Outer = _outer
 
   val reset = IO(Input(Bool())).suggestName("reset")

fpga/src/main/scala/vcu118/bringup/TestHarness.scala

@@ -5,7 +5,7 @@ import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config._
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink._
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.ip.xilinx._
 import sifive.fpgashells.shell._

generators/chipyard/src/main/scala/ChipTop.scala

@@ -5,7 +5,6 @@ import chisel3._
 import scala.collection.mutable.{ArrayBuffer}
 
 import freechips.rocketchip.prci.{ClockGroupIdentityNode, ClockSinkParameters, ClockSinkNode, ClockGroup}
-import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import org.chipsalliance.cde.config.{Parameters, Field}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike, BindingScope}
 import freechips.rocketchip.util.{DontTouch}

generators/chipyard/src/main/scala/DigitalTop.scala

@@ -40,7 +40,6 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
 }
 
 class DigitalTopModule[+L <: DigitalTop](l: L) extends ChipyardSystemModule(l)
-  with testchipip.cosim.CanHaveTraceIOModuleImp
   with sifive.blocks.devices.i2c.HasPeripheryI2CModuleImp
   with sifive.blocks.devices.pwm.HasPeripheryPWMModuleImp
   with sifive.blocks.devices.uart.HasPeripheryUARTModuleImp

generators/chipyard/src/main/scala/SpikeTile.scala

@@ -77,14 +77,15 @@ case class SpikeTileAttachParams(
 }
 
 case class SpikeTileParams(
-  hartId: Int = 0,
+  tileId: Int = 0,
   val core: SpikeCoreParams = SpikeCoreParams(),
   icacheParams: ICacheParams = ICacheParams(nWays = 32),
   dcacheParams: DCacheParams = DCacheParams(nWays = 32),
   tcmParams: Option[MasterPortParams] = None // tightly coupled memory
 ) extends InstantiableTileParams[SpikeTile]
 {
-  val name = Some("spike_tile")
+  val baseName = "spike_tile"
+  val uniqueName = s"${baseName}_$tileId"
   val beuAddr = None
   val blockerCtrlAddr = None
   val btb = None
@@ -92,7 +93,7 @@ case class SpikeTileParams(
   val dcache = Some(dcacheParams)
   val icache = Some(icacheParams)
   val clockSinkParams = ClockSinkParameters()
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): SpikeTile = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): SpikeTile = {
     new SpikeTile(this, crossing, lookup)
   }
 }
@@ -106,11 +107,11 @@ class SpikeTile(
     with SourcesExternalNotifications
 {
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: SpikeTileParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: SpikeTileParams, crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   // Required TileLink nodes
-  val intOutwardNode = IntIdentityNode()
+  val intOutwardNode = None
   val masterNode = visibilityNode
   val slaveNode = TLIdentityNode()
 
@@ -129,21 +130,21 @@ class SpikeTile(
   }
 
   ResourceBinding {
-    Resource(cpuDevice, "reg").bind(ResourceAddress(hartId))
+    Resource(cpuDevice, "reg").bind(ResourceAddress(tileId))
   }
 
 
   val icacheNode = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(
     sourceId = IdRange(0, 1),
-    name = s"Core ${staticIdForMetadataUseOnly} ICache")))))
+    name = s"Core ${tileId} ICache")))))
 
   val dcacheNode = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} DCache",
+    name          = s"Core ${tileId} DCache",
     sourceId      = IdRange(0, tileParams.dcache.get.nMSHRs),
     supportsProbe = TransferSizes(p(CacheBlockBytes), p(CacheBlockBytes)))))))
 
   val mmioNode = TLClientNode((Seq(TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} MMIO",
+    name          = s"Core ${tileId} MMIO",
     sourceId      = IdRange(0, 1),
     requestFifo   = true))))))
 
@@ -313,7 +314,7 @@ class SpikeBlackBox(
 }
 
 class SpikeTileModuleImp(outer: SpikeTile) extends BaseTileModuleImp(outer) {
-
+  val tileParams = outer.tileParams
   // We create a bundle here and decode the interrupt.
   val int_bundle = Wire(new TileInterrupts())
   outer.decodeCoreInterrupts(int_bundle)
@@ -337,7 +338,7 @@ class SpikeTileModuleImp(outer: SpikeTile) extends BaseTileModuleImp(outer) {
   // then the DTM-based bringup with SimDTM will be used. This isn't required to be
   // true, but it usually is
   val useDTM = p(ExportDebug).protocols.contains(DMI)
-  val spike = Module(new SpikeBlackBox(hartId, isaDTS, tileParams.core.nPMPs,
+  val spike = Module(new SpikeBlackBox(outer.tileId, outer.isaDTS, tileParams.core.nPMPs,
     tileParams.icache.get.nSets, tileParams.icache.get.nWays,
     tileParams.dcache.get.nSets, tileParams.dcache.get.nWays,
     tileParams.dcache.get.nMSHRs,
@@ -467,19 +468,21 @@ class SpikeTileModuleImp(outer: SpikeTile) extends BaseTileModuleImp(outer) {
   }
 }
 
-class WithNSpikeCores(n: Int = 1, tileParams: SpikeTileParams = SpikeTileParams(),
+class WithNSpikeCores(n: Int = 1, tileParams: SpikeTileParams = SpikeTileParams()
-  overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
+) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     // Calculate the next available hart ID (since hart ID cannot be duplicated)
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     // Create TileAttachParams for every core to be instantiated
     (0 until n).map { i =>
       SpikeTileAttachParams(
-        tileParams = tileParams.copy(hartId = i + idOffset)
+        tileParams = tileParams.copy(tileId = i + idOffset)
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
+
 })
 
 class WithSpikeTCM extends Config((site, here, up) => {
@@ -492,5 +495,5 @@ class WithSpikeTCM extends Config((site, here, up) => {
     )))
   }
   case ExtMem => None
-  case BankedL2Key => up(BankedL2Key).copy(nBanks = 0)
+  case SubsystemBankedCoherenceKey => up(SubsystemBankedCoherenceKey).copy(nBanks = 0)
 })

generators/chipyard/src/main/scala/Subsystem.scala

@@ -71,18 +71,24 @@ trait CanHaveChosenInDTS { this: BaseSubsystem =>
 }
 
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
-  with HasTiles
+    with InstantiatesHierarchicalElements
-  with HasPeripheryDebug
+    with HasTileNotificationSinks
-  with CanHaveHTIF
+    with HasTileInputConstants
-  with CanHaveChosenInDTS
+    with CanHavePeripheryCLINT
+    with CanHavePeripheryPLIC
+    with HasPeripheryDebug
+    with HasHierarchicalElementsRootContext
+    with HasHierarchicalElements
+    with CanHaveHTIF
+    with CanHaveChosenInDTS
 {
-  def coreMonitorBundles = tiles.map {
+  def coreMonitorBundles = totalTiles.values.map {
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
 
   // No-tile configs have to be handled specially.
-  if (tiles.size == 0) {
+  if (totalTiles.size == 0) {
     // no PLIC, so sink interrupts to nowhere
     require(!p(PLICKey).isDefined)
     val intNexus = IntNexusNode(sourceFn = x => x.head, sinkFn = x => x.head)
@@ -90,16 +96,12 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
     intSink := intNexus :=* ibus.toPLIC
 
     // avoids a bug when there are no interrupt sources
-    ibus.fromAsync := NullIntSource()
+    ibus { ibus.fromAsync := NullIntSource() }
 
     // Need to have at least 1 driver to the tile notification sinks
     tileHaltXbarNode := IntSourceNode(IntSourcePortSimple())
     tileWFIXbarNode := IntSourceNode(IntSourcePortSimple())
     tileCeaseXbarNode := IntSourceNode(IntSourcePortSimple())
-
-    // Sink reset vectors to nowhere
-    val resetVectorSink = BundleBridgeSink[UInt](Some(() => UInt(28.W)))
-    resetVectorSink := tileResetVectorNode
   }
 
   // Relying on [[TLBusWrapperConnection]].driveClockFromMaster for
@@ -107,7 +109,7 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   // ClockGroup. This makes it impossible to determine which clocks are driven
   // by which bus based on the member names, which is problematic when there is
   // a rational crossing between two buses. Instead, provide all bus clocks
-  // directly from the asyncClockGroupsNode in the subsystem to ensure bus
+  // directly from the allClockGroupsNode in the subsystem to ensure bus
   // names are always preserved in the top-level clock names.
   //
   // For example, using a RationalCrossing between the Sbus and Cbus, and
@@ -116,12 +118,12 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   // Conversly, if an async crossing is used, they instead receive names of the
   // form "subsystem_cbus_[0-9]*". The assignment below provides the latter names in all cases.
   Seq(PBUS, FBUS, MBUS, CBUS).foreach { loc =>
-    tlBusWrapperLocationMap.lift(loc).foreach { _.clockGroupNode := asyncClockGroupsNode }
+    tlBusWrapperLocationMap.lift(loc).foreach { _.clockGroupNode := allClockGroupsNode }
   }
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
 class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-  with HasTilesModuleImp
+    with HasHierarchicalElementsRootContextModuleImp
 {
 }

generators/chipyard/src/main/scala/System.scala

@@ -32,13 +32,6 @@ class ChipyardSystem(implicit p: Parameters) extends ChipyardSubsystem
   val bootROM  = p(BootROMLocated(location)).map { BootROM.attach(_, this, CBUS) }
   val maskROMs = p(MaskROMLocated(location)).map { MaskROM.attach(_, this, CBUS) }
 
-  // If there is no bootrom, the tile reset vector bundle will be tied to zero
-  if (bootROM.isEmpty) {
-    val fakeResetVectorSourceNode = BundleBridgeSource[UInt]()
-    InModuleBody { fakeResetVectorSourceNode.bundle := 0.U }
-    tileResetVectorNexusNode := fakeResetVectorSourceNode
-  }
-
   override lazy val module = new ChipyardSystemModule(this)
 }
 

generators/chipyard/src/main/scala/TestSuites.scala

@@ -65,7 +65,7 @@ class TestSuiteHelper
   */
   def addGenericTestSuites(tiles: Seq[TileParams])(implicit p: Parameters) = {
     val xlen = p(XLen)
-    tiles.find(_.hartId == 0).map { tileParams =>
+    tiles.find(_.tileId == 0).map { tileParams =>
       val coreParams = tileParams.core
       val vm = coreParams.useVM
       val env = if (vm) List("p","v") else List("p")

generators/chipyard/src/main/scala/clocking/ClockBinders.scala

@@ -18,16 +18,6 @@ class WithPLLSelectorDividerClockGenerator extends OverrideLazyIOBinder({
   (system: HasChipyardPRCI) => {
     // Connect the implicit clock
     implicit val p = GetSystemParameters(system)
-    val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
-    system.connectImplicitClockSinkNode(implicitClockSinkNode)
-    InModuleBody {
-      val implicit_clock = implicitClockSinkNode.in.head._1.clock
-      val implicit_reset = implicitClockSinkNode.in.head._1.reset
-      system.asInstanceOf[BaseSubsystem].module match { case l: LazyModuleImp => {
-        l.clock := implicit_clock
-        l.reset := implicit_reset
-      }}
-    }
     val tlbus = system.asInstanceOf[BaseSubsystem].locateTLBusWrapper(system.prciParams.slaveWhere)
     val baseAddress = system.prciParams.baseAddress
     val clockDivider  = system.prci_ctrl_domain { LazyModule(new TLClockDivider (baseAddress + 0x20000, tlbus.beatBytes)) }
@@ -38,7 +28,7 @@ class WithPLLSelectorDividerClockGenerator extends OverrideLazyIOBinder({
     clockSelector.tlNode := system.prci_ctrl_domain { TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := system.prci_ctrl_bus.get }
     pllCtrl.tlNode       := system.prci_ctrl_domain { TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := system.prci_ctrl_bus.get }
 
-    system.allClockGroupsNode := clockDivider.clockNode := clockSelector.clockNode
+    system.chiptopClockGroupsNode := clockDivider.clockNode := clockSelector.clockNode
 
     // Connect all other requested clocks
     val slowClockSource = ClockSourceNode(Seq(ClockSourceParameters()))
@@ -83,23 +73,12 @@ class WithPLLSelectorDividerClockGenerator extends OverrideLazyIOBinder({
 // This passes all clocks through to the TestHarness
 class WithPassthroughClockGenerator extends OverrideLazyIOBinder({
   (system: HasChipyardPRCI) => {
-    // Connect the implicit clock
     implicit val p = GetSystemParameters(system)
-    val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
-    system.connectImplicitClockSinkNode(implicitClockSinkNode)
-    InModuleBody {
-      val implicit_clock = implicitClockSinkNode.in.head._1.clock
-      val implicit_reset = implicitClockSinkNode.in.head._1.reset
-      system.asInstanceOf[BaseSubsystem].module match { case l: LazyModuleImp => {
-        l.clock := implicit_clock
-        l.reset := implicit_reset
-      }}
-    }
 
     // This aggregate node should do nothing
     val clockGroupAggNode = ClockGroupAggregateNode("fake")
     val clockGroupsSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
-    system.allClockGroupsNode := clockGroupAggNode := clockGroupsSourceNode
+    system.chiptopClockGroupsNode := clockGroupAggNode := clockGroupsSourceNode
 
     InModuleBody {
       val reset_io = IO(Input(AsyncReset()))

generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala

@@ -30,15 +30,14 @@ case class ChipyardPRCIControlParams(
 
 case object ChipyardPRCIControlKey extends Field[ChipyardPRCIControlParams](ChipyardPRCIControlParams())
 
-trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles =>
+trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesHierarchicalElements =>
-  require(p(SubsystemDriveAsyncClockGroupsKey).isEmpty, "Subsystem asyncClockGroups must be undriven")
+  require(!p(SubsystemDriveClockGroupsFromIO), "Subsystem allClockGroups cannot be driven from implicit clocks")
 
   val prciParams = p(ChipyardPRCIControlKey)
 
   // Set up clock domain
   private val tlbus = locateTLBusWrapper(prciParams.slaveWhere)
-  val prci_ctrl_domain = LazyModule(new ClockSinkDomain(name=Some("chipyard-prci-control")))
+  val prci_ctrl_domain = tlbus.generateSynchronousDomain.suggestName("chipyard_prcictrl_domain")
-  prci_ctrl_domain.clockNode := tlbus.fixedClockNode
 
   val prci_ctrl_bus = Option.when(prciParams.generatePRCIXBar) { prci_ctrl_domain { TLXbar() } }
   prci_ctrl_bus.foreach(xbar => tlbus.coupleTo("prci_ctrl") { (xbar
@@ -49,29 +48,13 @@ trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles =>
 
   // Aggregate all the clock groups into a single node
   val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
-  val allClockGroupsNode = ClockGroupEphemeralNode()
 
-  // There are two "sets" of clocks which must be dealt with
+  // The diplomatic clocks in the subsystem are routed to this allClockGroupsNode
-
-  // 1. The implicit clock from the subsystem. RC is moving away from depending on this
-  //    clock, but some modules still use it. Since the implicit clock sink node
-  //    is created in the ChipTop (the hierarchy wrapping the subsystem), this function
-  //    is provided to allow connecting that clock to the clock aggregator. This function
-  //    should be called in the ChipTop context
-  def connectImplicitClockSinkNode(sink: ClockSinkNode) = {
-    val implicitClockGrouper = this { ClockGroup() }
-    (sink
-      := implicitClockGrouper
-      := aggregator)
-  }
-
-  // 2. The rest of the diplomatic clocks in the subsystem are routed to this asyncClockGroupsNode
   val clockNamePrefixer = ClockGroupNamePrefixer()
-  (asyncClockGroupsNode
+  (allClockGroupsNode
     :*= clockNamePrefixer
     :*= aggregator)
 
-
   // Once all the clocks are gathered in the aggregator node, several steps remain
   // 1. Assign frequencies to any clock groups which did not specify a frequency.
   // 2. Combine duplicated clock groups (clock groups which physically should be in the same clock domain)
@@ -92,7 +75,7 @@ trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles =>
   } }
   val tileResetSetter    = Option.when(prciParams.enableTileResetSetting) { prci_ctrl_domain {
     val reset_setter = LazyModule(new TileResetSetter(prciParams.baseAddress + 0x10000, tlbus.beatBytes,
-      tile_prci_domains.map(_.tile_reset_domain.clockNode.portParams(0).name.get), Nil))
+      tile_prci_domains.map(_._2.tile_reset_domain.clockNode.portParams(0).name.get).toSeq, Nil))
     reset_setter.tlNode := TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := prci_ctrl_bus.get
     reset_setter
   } }
@@ -116,11 +99,14 @@ RTL SIMULATORS, NAMELY VERILATOR.
 """ + Console.RESET)
   }
 
+  // The chiptopClockGroupsNode shouuld be what ClockBinders attach to
+  val chiptopClockGroupsNode = ClockGroupEphemeralNode()
+
   (aggregator
     := frequencySpecifier
     := clockGroupCombiner
     := resetSynchronizer
     := tileClockGater.map(_.clockNode).getOrElse(ClockGroupEphemeralNode()(ValName("temp")))
     := tileResetSetter.map(_.clockNode).getOrElse(ClockGroupEphemeralNode()(ValName("temp")))
-    := allClockGroupsNode)
+    := chiptopClockGroupsNode)
 }

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -55,6 +55,7 @@ class AbstractConfig extends Config(
   new chipyard.clocking.WithPassthroughClockGenerator ++
   new chipyard.clocking.WithClockGroupsCombinedByName(("uncore", Seq("sbus", "mbus", "pbus", "fbus", "cbus", "obus", "implicit"), Seq("tile"))) ++
   new chipyard.config.WithPeripheryBusFrequency(500.0) ++           // Default 500 MHz pbus
+  new chipyard.config.WithControlBusFrequency(500.0) ++             // Default 500 MHz cbus
   new chipyard.config.WithMemoryBusFrequency(500.0) ++              // Default 500 MHz mbus
   new chipyard.config.WithControlBusFrequency(500.0) ++             // Default 500 MHz cbus
   new chipyard.config.WithSystemBusFrequency(500.0) ++              // Default 500 MHz sbus
@@ -75,7 +76,7 @@ class AbstractConfig extends Config(
   new chipyard.config.WithBootROM ++                                // use default bootrom
   new chipyard.config.WithUART ++                                   // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                           // use L2 TLBs
-  new chipyard.config.WithNoSubsystemDrivenClocks ++                // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
+  new chipyard.config.WithNoSubsystemClockIO ++                     // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
   new chipyard.config.WithInheritBusFrequencyAssignments ++         // Unspecified clocks within a bus will receive the bus frequency if set
   new freechips.rocketchip.subsystem.WithNMemoryChannels(1) ++      // Default 1 memory channels
   new freechips.rocketchip.subsystem.WithClockGateModel ++          // add default EICG_wrapper clock gate model

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -2,6 +2,7 @@ package chipyard
 
 import org.chipsalliance.cde.config.{Config}
 import freechips.rocketchip.diplomacy.{AsynchronousCrossing}
+import freechips.rocketchip.subsystem.{InCluster}
 
 // --------------
 // Rocket Configs
@@ -88,3 +89,10 @@ class PrefetchingRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNonblockingL1(2) ++                           // non-blocking L1D$, L1 prefetching only works with non-blocking L1D$
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++                               // single rocket-core
   new chipyard.config.AbstractConfig)
+
+class ClusteredRocketConfig extends Config(
+  new freechips.rocketchip.subsystem.WithNBigCores(4, location=InCluster(1)) ++
+  new freechips.rocketchip.subsystem.WithNBigCores(4, location=InCluster(0)) ++
+  new freechips.rocketchip.subsystem.WithCluster(1) ++
+  new freechips.rocketchip.subsystem.WithCluster(0) ++
+  new chipyard.config.AbstractConfig)

generators/chipyard/src/main/scala/config/TracegenConfigs.scala

@@ -12,10 +12,11 @@ class AbstractTraceGenConfig extends Config(
   new chipyard.iobinders.WithAXI4MemPunchthrough ++
   new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.clocking.WithPassthroughClockGenerator ++
-  new chipyard.clocking.WithClockGroupsCombinedByName(("uncore", Seq("sbus", "implicit"), Nil)) ++
+  new chipyard.clocking.WithClockGroupsCombinedByName(("uncore", Seq("sbus"), Nil)) ++
   new chipyard.config.WithTracegenSystem ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
+  new chipyard.config.WithNoSubsystemClockIO ++
   new chipyard.config.WithMemoryBusFrequency(1000.0) ++
+  new chipyard.config.WithControlBusFrequency(1000.0) ++
   new chipyard.config.WithSystemBusFrequency(1000.0) ++
   new chipyard.config.WithPeripheryBusFrequency(1000.0) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++

generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala

@@ -19,8 +19,8 @@ import testchipip.soc.{OffchipBusKey}
 // with the implicit clocks of Subsystem. Don't do that, instead we extend
 // the diplomacy graph upwards into the ChipTop, where we connect it to
 // our clock drivers
-class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
+class WithNoSubsystemClockIO extends Config((site, here, up) => {
-  case SubsystemDriveAsyncClockGroupsKey => None
+  case SubsystemDriveClockGroupsFromIO => false
 })
 
 /**

generators/chipyard/src/main/scala/config/fragments/RoCCFragments.scala

@@ -12,15 +12,15 @@ import gemmini._
 import chipyard.{TestSuitesKey, TestSuiteHelper}
 
 /**
- * Map from a hartId to a particular RoCC accelerator
+ * Map from a tileId to a particular RoCC accelerator
  */
 case object MultiRoCCKey extends Field[Map[Int, Seq[Parameters => LazyRoCC]]](Map.empty[Int, Seq[Parameters => LazyRoCC]])
 
 /**
- * Config fragment to enable different RoCCs based on the hartId
+ * Config fragment to enable different RoCCs based on the tileId
  */
 class WithMultiRoCC extends Config((site, here, up) => {
-  case BuildRoCC => site(MultiRoCCKey).getOrElse(site(TileKey).hartId, Nil)
+  case BuildRoCC => site(MultiRoCCKey).getOrElse(site(TileKey).tileId, Nil)
 })
 
 /**
@@ -39,7 +39,7 @@ class WithMultiRoCCFromBuildRoCC(harts: Int*) extends Config((site, here, up) =>
  *
  * For ex:
  *   Core 0, 1, 2, 3 have been defined earlier
- *     with hartIds of 0, 1, 2, 3 respectively
+ *     with tileIds of 0, 1, 2, 3 respectively
  *   And you call WithMultiRoCCHwacha(0,1)
  *   Then Core 0 and 1 will get a Hwacha
  *

generators/chipyard/src/main/scala/config/fragments/SubsystemFragments.scala

@@ -7,7 +7,7 @@ import sifive.blocks.inclusivecache.{InclusiveCachePortParameters}
 
 // Replaces the L2 with a broadcast manager for maintaining coherence
 class WithBroadcastManager extends Config((site, here, up) => {
-  case BankedL2Key => up(BankedL2Key, site).copy(coherenceManager = CoherenceManagerWrapper.broadcastManager)
+  case SubsystemBankedCoherenceKey => up(SubsystemBankedCoherenceKey, site).copy(coherenceManager = CoherenceManagerWrapper.broadcastManager)
 })
 
 class WithBroadcastParams(params: BroadcastParams) extends Config((site, here, up) => {

generators/chipyard/src/main/scala/config/fragments/TileFragments.scala

@@ -78,7 +78,7 @@ class WithRocketCacheRowBits(rowBits: Int = 64) extends Config((site, here, up)
 class WithRocketICacheScratchpad extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
-      icache = tp.tileParams.icache.map(_.copy(itimAddr = Some(0x300000 + tp.tileParams.hartId * 0x10000)))
+      icache = tp.tileParams.icache.map(_.copy(itimAddr = Some(0x300000 + tp.tileParams.tileId * 0x10000)))
     ))
   }
 })
@@ -86,7 +86,7 @@ class WithRocketICacheScratchpad extends Config((site, here, up) => {
 class WithRocketDCacheScratchpad extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
-      dcache = tp.tileParams.dcache.map(_.copy(nSets = 32, nWays = 1, scratch = Some(0x200000 + tp.tileParams.hartId * 0x10000)))
+      dcache = tp.tileParams.dcache.map(_.copy(nSets = 32, nWays = 1, scratch = Some(0x200000 + tp.tileParams.tileId * 0x10000)))
     ))
   }
 })
@@ -94,15 +94,15 @@ class WithRocketDCacheScratchpad extends Config((site, here, up) => {
 class WithTilePrefetchers extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
-      master = TilePrefetchingMasterPortParams(tp.tileParams.hartId, tp.crossingParams.master)))
+      master = TilePrefetchingMasterPortParams(tp.tileParams.tileId, tp.crossingParams.master)))
     case tp: BoomTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
-      master = TilePrefetchingMasterPortParams(tp.tileParams.hartId, tp.crossingParams.master)))
+      master = TilePrefetchingMasterPortParams(tp.tileParams.tileId, tp.crossingParams.master)))
     case tp: SodorTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
-      master = TilePrefetchingMasterPortParams(tp.tileParams.hartId, tp.crossingParams.master)))
+      master = TilePrefetchingMasterPortParams(tp.tileParams.tileId, tp.crossingParams.master)))
     case tp: IbexTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
-      master = TilePrefetchingMasterPortParams(tp.tileParams.hartId, tp.crossingParams.master)))
+      master = TilePrefetchingMasterPortParams(tp.tileParams.tileId, tp.crossingParams.master)))
     case tp: CVA6TileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
-      master = TilePrefetchingMasterPortParams(tp.tileParams.hartId, tp.crossingParams.master)))
+      master = TilePrefetchingMasterPortParams(tp.tileParams.tileId, tp.crossingParams.master)))
   }
 })
 

generators/chipyard/src/main/scala/example/FlatChipTop.scala

@@ -30,9 +30,6 @@ class FlatChipTop(implicit p: Parameters) extends LazyModule with HasChipyardPor
   //========================
   // Diplomatic clock stuff
   //========================
-  val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
-  system.connectImplicitClockSinkNode(implicitClockSinkNode)
-
   val tlbus = system.locateTLBusWrapper(system.prciParams.slaveWhere)
   val baseAddress = system.prciParams.baseAddress
   val clockDivider  = system.prci_ctrl_domain { LazyModule(new TLClockDivider (baseAddress + 0x20000, tlbus.beatBytes)) }
@@ -43,7 +40,7 @@ class FlatChipTop(implicit p: Parameters) extends LazyModule with HasChipyardPor
   tlbus.coupleTo("clock-sel-ctrl") { clockSelector.tlNode := TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := TLBuffer() := _ }
   tlbus.coupleTo("pll-ctrl") { pllCtrl.tlNode := TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := TLBuffer() := _ }
 
-  system.allClockGroupsNode := clockDivider.clockNode := clockSelector.clockNode
+  system.chiptopClockGroupsNode := clockDivider.clockNode := clockSelector.clockNode
 
   // Connect all other requested clocks
   val slowClockSource = ClockSourceNode(Seq(ClockSourceParameters()))
@@ -69,13 +66,6 @@ class FlatChipTop(implicit p: Parameters) extends LazyModule with HasChipyardPor
     //=========================
     // Clock/reset
     //=========================
-    val implicit_clock = implicitClockSinkNode.in.head._1.clock
-    val implicit_reset = implicitClockSinkNode.in.head._1.reset
-    system.module match { case l: LazyModuleImp => {
-      l.clock := implicit_clock
-      l.reset := implicit_reset
-    }}
-
     val clock_wire = Wire(Input(Clock()))
     val reset_wire = Wire(Input(AsyncReset()))
     val (clock_pad, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, "clock", p(IOCellKey))

generators/chipyard/src/main/scala/example/GCD.scala

@@ -185,7 +185,7 @@ trait CanHavePeripheryGCD { this: BaseSubsystem =>
 // DOC include end: GCD lazy trait
 
 // DOC include start: GCD imp trait
-trait CanHavePeripheryGCDModuleImp extends LazyModuleImp {
+trait CanHavePeripheryGCDModuleImp extends LazyRawModuleImp {
   val outer: CanHavePeripheryGCD
   val gcd_busy = outer.gcd match {
     case Some(gcd) => {

generators/chipyard/src/main/scala/example/TutorialTile.scala

@@ -82,7 +82,7 @@ case class MyTileAttachParams(
 
 case class MyTileParams(
   name: Option[String] = Some("my_tile"),
-  hartId: Int = 0,
+  tileId: Int = 0,
   trace: Boolean = false,
   val core: MyCoreParams = MyCoreParams()
 ) extends InstantiableTileParams[MyTile]
@@ -94,9 +94,11 @@ case class MyTileParams(
   val dcache: Option[DCacheParams] = Some(DCacheParams())
   val icache: Option[ICacheParams] = Some(ICacheParams())
   val clockSinkParams: ClockSinkParameters = ClockSinkParameters()
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): MyTile = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): MyTile = {
     new MyTile(this, crossing, lookup)
   }
+  val baseName = name.getOrElse("my_tile")
+  val uniqueName = s"${baseName}_$tileId"
 }
 
 // DOC include start: Tile class
@@ -111,11 +113,11 @@ class MyTile(
 {
 
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: MyTileParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: MyTileParams, crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   // Require TileLink nodes
-  val intOutwardNode = IntIdentityNode()
+  val intOutwardNode = None
   val masterNode = visibilityNode
   val slaveNode = TLIdentityNode()
 
@@ -135,7 +137,7 @@ class MyTile(
   }
 
   ResourceBinding {
-    Resource(cpuDevice, "reg").bind(ResourceAddress(hartId))
+    Resource(cpuDevice, "reg").bind(ResourceAddress(tileId))
   }
 
   // TODO: Create TileLink nodes and connections here.
@@ -228,15 +230,15 @@ class MyTileModuleImp(outer: MyTile) extends BaseTileModuleImp(outer){
 }
 
 // DOC include start: Config fragment
-class WithNMyCores(n: Int = 1, overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
+class WithNMyCores(n: Int = 1) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     // Calculate the next available hart ID (since hart ID cannot be duplicated)
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     // Create TileAttachParams for every core to be instantiated
     (0 until n).map { i =>
       MyTileAttachParams(
-        tileParams = MyTileParams(hartId = i + idOffset),
+        tileParams = MyTileParams(tileId = i + idOffset),
         crossingParams = RocketCrossingParams()
       )
     } ++ prev
@@ -245,5 +247,6 @@ class WithNMyCores(n: Int = 1, overrideIdOffset: Option[Int] = None) extends Con
   case SystemBusKey => up(SystemBusKey, site).copy(beatBytes = 8)
   // The # of instruction bits. Use maximum # of bits if your core supports both 32 and 64 bits.
   case XLen => 64
+  case NumTiles => up(NumTiles) + n
 })
 // DOC include end: Config fragment

generators/chipyard/src/main/scala/example/dsptools/GenericFIR.scala

@@ -199,12 +199,13 @@ class TLGenericFIRChain[T<:Data:Ring] (genIn: T, genOut: T, coeffs: => Seq[T], p
 trait CanHavePeripheryStreamingFIR extends BaseSubsystem {
   val streamingFIR = p(GenericFIRKey) match {
     case Some(params) => {
-      val streamingFIR = LazyModule(new TLGenericFIRChain(
+      val domain = pbus.generateSynchronousDomain.suggestName("fir_domain")
+      val streamingFIR = domain { LazyModule(new TLGenericFIRChain(
         genIn = FixedPoint(8.W, 3.BP),
         genOut = FixedPoint(8.W, 3.BP),
         coeffs = Seq(1.U.asFixedPoint(0.BP), 2.U.asFixedPoint(0.BP), 3.U.asFixedPoint(0.BP)),
-        params = params))
+        params = params)) }
-      pbus.coupleTo("streamingFIR") { streamingFIR.mem.get := TLFIFOFixer() := TLFragmenter(pbus.beatBytes, pbus.blockBytes) := _ }
+      pbus.coupleTo("streamingFIR") { domain { streamingFIR.mem.get := TLFIFOFixer() := TLFragmenter(pbus.beatBytes, pbus.blockBytes) } := _ }
       Some(streamingFIR)
     }
     case None => None

generators/chipyard/src/main/scala/example/dsptools/StreamingPassthrough.scala

@@ -131,8 +131,9 @@ class TLStreamingPassthroughChain[T<:Data:Ring](params: StreamingPassthroughPara
 trait CanHavePeripheryStreamingPassthrough { this: BaseSubsystem =>
   val passthrough = p(StreamingPassthroughKey) match {
     case Some(params) => {
-      val streamingPassthroughChain = LazyModule(new TLStreamingPassthroughChain(params, UInt(32.W)))
+      val domain = pbus.generateSynchronousDomain.suggestName("streaming_passthrough_domain")
-      pbus.coupleTo("streamingPassthrough") { streamingPassthroughChain.mem.get := TLFIFOFixer() := TLFragmenter(pbus.beatBytes, pbus.blockBytes) := _ }
+      val streamingPassthroughChain = domain { LazyModule(new TLStreamingPassthroughChain(params, UInt(32.W))) }
+      pbus.coupleTo("streamingPassthrough") { domain { streamingPassthroughChain.mem.get := TLFIFOFixer() := TLFragmenter(pbus.beatBytes, pbus.blockBytes)} := _ }
       Some(streamingPassthroughChain)
     }
     case None => None

generators/chipyard/src/main/scala/iobinders/IOBinders.scala

@@ -29,7 +29,7 @@ import testchipip.spi.{SPIChipIO}
 import testchipip.boot.{CanHavePeripheryCustomBootPin}
 import testchipip.util.{ClockedIO}
 import testchipip.iceblk.{CanHavePeripheryBlockDevice, BlockDeviceKey, BlockDeviceIO}
-import testchipip.cosim.{CanHaveTraceIOModuleImp, TraceOutputTop, SpikeCosimConfig}
+import testchipip.cosim.{CanHaveTraceIO, TraceOutputTop, SpikeCosimConfig}
 import testchipip.tsi.{CanHavePeripheryUARTTSI, UARTTSIIO}
 import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
 import chipyard.{CanHaveMasterTLMemPort, ChipyardSystem, ChipyardSystemModule}
@@ -455,14 +455,14 @@ class WithTraceGenSuccessPunchthrough extends OverrideIOBinder({
   }
 })
 
-class WithTraceIOPunchthrough extends OverrideIOBinder({
+class WithTraceIOPunchthrough extends OverrideLazyIOBinder({
-  (system: CanHaveTraceIOModuleImp) => {
+  (system: CanHaveTraceIO) => InModuleBody {
     val ports: Option[TracePort] = system.traceIO.map { t =>
       val trace = IO(DataMirror.internal.chiselTypeClone[TraceOutputTop](t)).suggestName("trace")
       trace <> t
       val p = GetSystemParameters(system)
-      val chipyardSystem = system.asInstanceOf[ChipyardSystemModule[_]].outer.asInstanceOf[ChipyardSystem]
+      val chipyardSystem = system.asInstanceOf[ChipyardSystem]
-      val tiles = chipyardSystem.tiles
+      val tiles = chipyardSystem.totalTiles.values
       val cfg = SpikeCosimConfig(
         isa = tiles.headOption.map(_.isaDTS).getOrElse(""),
         vlen = tiles.headOption.map(_.tileParams.core.vLen).getOrElse(0),
@@ -511,8 +511,8 @@ class WithDontTouchPorts extends OverrideIOBinder({
 })
 
 class WithNMITiedOff extends ComposeIOBinder({
-  (system: HasTilesModuleImp) => {
+  (system: HasHierarchicalElementsRootContextModuleImp) => {
-    system.nmi.flatten.foreach { nmi =>
+    system.nmi.foreach { nmi =>
       nmi.rnmi := false.B
       nmi.rnmi_interrupt_vector := 0.U
       nmi.rnmi_exception_vector := 0.U

generators/firechip/src/main/scala/FireSim.scala

@@ -8,7 +8,7 @@ import chisel3._
 import chisel3.experimental.{IO, annotate}
 
 import freechips.rocketchip.prci._
-import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey, HasTiles}
+import freechips.rocketchip.subsystem._
 import org.chipsalliance.cde.config.{Field, Config, Parameters}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, InModuleBody, ValName}
 import freechips.rocketchip.util.{ResetCatchAndSync, RecordMap}
@@ -103,8 +103,8 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessInsta
   // FireSim ModelMultithreading
   chiptops.foreach {
     case c: ChipTop => c.lazySystem match {
-      case ls: HasTiles => {
+      case ls: InstantiatesHierarchicalElements => {
-        if (p(FireSimMultiCycleRegFile)) ls.tiles.map {
+        if (p(FireSimMultiCycleRegFile)) ls.totalTiles.values.map {
           case r: RocketTile => {
             annotate(MemModelAnnotation(r.module.core.rocketImpl.rf.rf))
             r.module.fpuOpt.foreach(fpu => annotate(MemModelAnnotation(fpu.fpuImpl.regfile)))
@@ -120,7 +120,7 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessInsta
           }
           case _ =>
         }
-        if (p(FireSimFAME5)) ls.tiles.map {
+        if (p(FireSimFAME5)) ls.totalTiles.values.map {
           case b: BoomTile =>
             annotate(EnableModelMultiThreadingAnnotation(b.module))
           case r: RocketTile =>

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -122,6 +122,7 @@ class WithFireSimHighPerfClocking extends Config(
   // This frequency selection matches FireSim's legacy selection and is required
   // to support 200Gb NIC performance. You may select a smaller value.
   new chipyard.config.WithPeripheryBusFrequency(3200.0) ++
+  new chipyard.config.WithControlBusFrequency(3200.0) ++
   new chipyard.config.WithSystemBusFrequency(3200.0) ++
   new chipyard.config.WithFrontBusFrequency(3200.0) ++
   new chipyard.config.WithControlBusFrequency(3200.0) ++
@@ -142,6 +143,7 @@ class WithFireSimConfigTweaks extends Config(
   new chipyard.config.WithSystemBusFrequency(1000.0) ++
   new chipyard.config.WithControlBusFrequency(1000.0) ++
   new chipyard.config.WithPeripheryBusFrequency(1000.0) ++
+  new chipyard.config.WithControlBusFrequency(1000.0) ++
   new chipyard.config.WithMemoryBusFrequency(1000.0) ++
   new chipyard.config.WithFrontBusFrequency(1000.0) ++
   new WithFireSimDesignTweaks

generators/tracegen/src/main/scala/Configs.scala

@@ -13,19 +13,18 @@ import scala.math.{max, min}
 
 class WithTraceGen(
   n: Int = 2,
-  overrideIdOffset: Option[Int] = None,
   overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
     params.zipWithIndex.map { case (dcp, i) =>
       TraceGenTileAttachParams(
         tileParams = TraceGenParams(
-          hartId = i + idOffset,
+          tileId = i + idOffset,
           dcache = Some(dcp),
           wordBits = site(XLen),
           addrBits = 48,
@@ -48,23 +47,23 @@ class WithTraceGen(
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
 
 class WithBoomTraceGen(
   n: Int = 2,
-  overrideIdOffset: Option[Int] = None,
   overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nMSHRs = 4, nSets = 16, nWays = 2) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
     params.zipWithIndex.map { case (dcp, i) =>
       BoomTraceGenTileAttachParams(
         tileParams = BoomTraceGenParams(
-          hartId = i + idOffset,
+          tileId = i + idOffset,
           dcache = Some(dcp),
           wordBits = site(XLen),
           addrBits = 48,
@@ -84,24 +83,24 @@ class WithBoomTraceGen(
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
 
 class WithL2TraceGen(
   n: Int = 2,
-  overrideIdOffset: Option[Int] = None,
   overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
 
     params.zipWithIndex.map { case (dcp, i) =>
       TraceGenTileAttachParams(
         tileParams = TraceGenParams(
-          hartId = i + idOffset,
+          tileId = i + idOffset,
           dcache = Some(dcp),
           wordBits = site(XLen),
           addrBits = 48,
@@ -126,4 +125,5 @@ class WithL2TraceGen(
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })

generators/tracegen/src/main/scala/System.scala

@@ -9,15 +9,28 @@ import freechips.rocketchip.subsystem._
 import boom.lsu.BoomTraceGenTile
 
 class TraceGenSystem(implicit p: Parameters) extends BaseSubsystem
-    with HasTiles
+    with InstantiatesHierarchicalElements
+    with HasTileNotificationSinks
+    with HasTileInputConstants
+    with HasHierarchicalElementsRootContext
+    with HasHierarchicalElements
     with CanHaveMasterAXI4MemPort {
 
   def coreMonitorBundles = Nil
-  val tileStatusNodes = tiles.collect {
+
+  val tileStatusNodes = totalTiles.values.toSeq.collect {
     case t: GroundTestTile => t.statusNode.makeSink()
     case t: BoomTraceGenTile => t.statusNode.makeSink()
   }
-  lazy val debugNode = IntSyncXbar() := NullIntSyncSource()
+
+  lazy val fakeClockDomain = sbus.generateSynchronousDomain
+
+  lazy val clintOpt = None
+  lazy val debugOpt = None
+  lazy val plicOpt = None
+  lazy val clintDomainOpt = Some(fakeClockDomain)
+  lazy val plicDomainOpt = Some(fakeClockDomain)
+
   override lazy val module = new TraceGenSystemModuleImp(this)
 }