Merge branch 'dev' into patch-2

generators/chipyard/src/main/scala/ChipTop.scala

@@ -15,6 +15,9 @@ import barstools.iocell.chisel._
 
 case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => new DigitalTop()(p))
 
+trait HasReferenceClockFreq {
+  def refClockFreqMHz: Double
+}
 
 /**
  * The base class used for building chips. This constructor instantiates a module specified by the BuildSystem parameter,
@@ -24,15 +27,16 @@ case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters)
  */
 
 class ChipTop(implicit p: Parameters) extends LazyModule with BindingScope
-    with HasTestHarnessFunctions with HasIOBinders {
+    with HasTestHarnessFunctions with HasReferenceClockFreq with HasIOBinders {
   // The system module specified by BuildSystem
   lazy val lazySystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
 
-  // The implicitClockSinkNode provides the implicit clock and reset for the System
+  // The implicitClockSinkNode provides the implicit clock and reset for the system (connected by clocking scheme)
   val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
 
   // Generate Clocks and Reset
-  p(ClockingSchemeKey)(this)
+  val mvRefClkFreq = p(ClockingSchemeKey)(this)
+  def refClockFreqMHz: Double = mvRefClkFreq.getWrappedValue
 
   // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children
   // of ChipTop (ex: ClockGroup) do not receive clock or reset.

generators/chipyard/src/main/scala/Clocks.scala

@@ -7,25 +7,26 @@ import scala.collection.mutable.{ArrayBuffer}
 import freechips.rocketchip.prci._
 import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey, InstantiatesTiles}
 import freechips.rocketchip.config.{Parameters, Field, Config}
-import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody, LazyModule}
+import freechips.rocketchip.diplomacy.{ModuleValue, OutwardNodeHandle, InModuleBody, LazyModule}
 import freechips.rocketchip.util.{ResetCatchAndSync}
 
 import barstools.iocell.chisel._
 import testchipip.{TLTileResetCtrl}
 
 import chipyard.clocking._
+import chipyard.iobinders._
 
 /**
  * A simple reset implementation that punches out reset ports
- * for standard Module classes. Three basic reset schemes
- * are provided. See [[GlobalResetScheme]].
+ * for standard Module classes. The ChipTop reset pin is Async.
+ * Synchronization is performed in the ClockGroupResetSynchronizer
  */
 object GenerateReset {
   def apply(chiptop: ChipTop, clock: Clock): Reset = {
     implicit val p = chiptop.p
     // this needs directionality so generateIOFromSignal works
     val async_reset_wire = Wire(Input(AsyncReset()))
-    val (reset_io, resetIOCell) = IOCell.generateIOFromSignal(async_reset_wire, "reset",
+    val (reset_io, resetIOCell) = IOCell.generateIOFromSignal(async_reset_wire, "reset", p(IOCellKey),
       abstractResetAsAsync = true)
 
     chiptop.iocells ++= resetIOCell
@@ -38,7 +39,7 @@ object GenerateReset {
 }
 
 
-case object ClockingSchemeKey extends Field[ChipTop => Unit](ClockingSchemeGenerators.dividerOnlyClockGenerator)
+case object ClockingSchemeKey extends Field[ChipTop => ModuleValue[Double]](ClockingSchemeGenerators.dividerOnlyClockGenerator)
 /*
   * This is a Seq of assignment functions, that accept a clock name and return an optional frequency.
   * Functions that appear later in this seq have higher precedence that earlier ones.
@@ -59,7 +60,7 @@ class ClockNameContainsAssignment(name: String, fMHz: Double) extends Config((si
 })
 
 object ClockingSchemeGenerators {
-  val dividerOnlyClockGenerator: ChipTop => Unit = { chiptop =>
+  val dividerOnlyClockGenerator: ChipTop => ModuleValue[Double] = { chiptop =>
     implicit val p = chiptop.p
 
     // Requires existence of undriven asyncClockGroups in subsystem
@@ -70,31 +71,40 @@ object ClockingSchemeGenerators {
 
     // Add a control register for each tile's reset
     val resetSetter = chiptop.lazySystem match {
-      case sys: BaseSubsystem with InstantiatesTiles => TLTileResetCtrl(sys)
-      case _ => ClockGroupEphemeralNode()
+      case sys: BaseSubsystem with InstantiatesTiles => Some(TLTileResetCtrl(sys))
+      case _ => None
     }
+    val resetSetterResetProvider = resetSetter.map(_.tileResetProviderNode).getOrElse(ClockGroupEphemeralNode())
 
     val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
+    // provides the implicit clock to the system
     (chiptop.implicitClockSinkNode
       := ClockGroup()
       := aggregator)
+    // provides the system clock (ex. the bus clocks)
     (systemAsyncClockGroup
-      :*= resetSetter
       :*= ClockGroupNamePrefixer()
       :*= aggregator)
 
     val referenceClockSource =  ClockSourceNode(Seq(ClockSourceParameters()))
+    val dividerOnlyClkGenerator = LazyModule(new DividerOnlyClockGenerator("buildTopClockGenerator"))
+    // provides all the divided clocks (from the top-level clock)
     (aggregator
       := ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey), p(DefaultClockFrequencyKey))
       := ClockGroupResetSynchronizer()
-      := DividerOnlyClockGenerator()
+      := resetSetterResetProvider
+      := dividerOnlyClkGenerator.node
       := referenceClockSource)
 
+    val asyncResetBroadcast = FixedClockBroadcast(None)
+    resetSetter.foreach(_.asyncResetSinkNode := asyncResetBroadcast)
+    val asyncResetSource = ClockSourceNode(Seq(ClockSourceParameters()))
+    asyncResetBroadcast := asyncResetSource
 
     InModuleBody {
       val clock_wire = Wire(Input(Clock()))
       val reset_wire = GenerateReset(chiptop, clock_wire)
-      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, "clock")
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, "clock", p(IOCellKey))
       chiptop.iocells ++= clockIOCell
 
       referenceClockSource.out.unzip._1.map { o =>
@@ -102,9 +112,17 @@ object ClockingSchemeGenerators {
         o.reset := reset_wire
       }
 
+      asyncResetSource.out.unzip._1.map { o =>
+        o.clock := false.B.asClock // async reset broadcast network does not provide a clock
+        o.reset := reset_wire
+      }
+
       chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
-        clock_io := th.harnessClock
+        clock_io := th.buildtopClock
         Nil })
+
+      // return the reference frequency
+      dividerOnlyClkGenerator.module.referenceFreq
     }
   }
 }

generators/chipyard/src/main/scala/ConfigFragments.scala

@@ -20,7 +20,7 @@ import testchipip._
 import tracegen.{TraceGenSystem}
 
 import hwacha.{Hwacha}
-import gemmini.{Gemmini, GemminiConfigs}
+import gemmini._
 
 import boom.common.{BoomTileAttachParams}
 import cva6.{CVA6TileAttachParams}
@@ -83,6 +83,17 @@ class WithMultiRoCC extends Config((site, here, up) => {
   case BuildRoCC => site(MultiRoCCKey).getOrElse(site(TileKey).hartId, Nil)
 })
 
+/**
+ * Assigns what was previously in the BuildRoCC key to specific harts with MultiRoCCKey
+ * Must be paired with WithMultiRoCC
+ */
+class WithMultiRoCCFromBuildRoCC(harts: Int*) extends Config((site, here, up) => {
+  case BuildRoCC => Nil
+  case MultiRoCCKey => up(MultiRoCCKey, site) ++ harts.distinct.map { i =>
+    (i -> up(BuildRoCC, site))
+  }
+})
+
 /**
  * Config fragment to add Hwachas to cores based on hart
  *
@@ -108,11 +119,12 @@ class WithMultiRoCCHwacha(harts: Int*) extends Config(
   })
 )
 
-class WithMultiRoCCGemmini(harts: Int*) extends Config((site, here, up) => {
+class WithMultiRoCCGemmini[T <: Data : Arithmetic, U <: Data, V <: Data](
+  harts: Int*)(gemminiConfig: GemminiArrayConfig[T,U,V] = GemminiConfigs.defaultConfig) extends Config((site, here, up) => {
   case MultiRoCCKey => up(MultiRoCCKey, site) ++ harts.distinct.map { i =>
     (i -> Seq((p: Parameters) => {
       implicit val q = p
-      val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig))
+      val gemmini = LazyModule(new Gemmini(gemminiConfig))
       gemmini
     }))
   }
@@ -139,6 +151,16 @@ class WithNPerfCounters(n: Int = 29) extends Config((site, here, up) => {
   }
 })
 
+class WithNPMPs(n: Int = 8) extends Config((site, here, up) => {
+  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+    case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
+      core = tp.tileParams.core.copy(nPMPs = n)))
+    case tp: BoomTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
+      core = tp.tileParams.core.copy(nPMPs = n)))
+    case other => other
+  }
+})
+
 class WithRocketICacheScratchpad extends Config((site, here, up) => {
   case RocketTilesKey => up(RocketTilesKey, site) map { r =>
     r.copy(icache = r.icache.map(_.copy(itimAddr = Some(0x300000 + r.hartId * 0x10000))))
@@ -193,7 +215,31 @@ class WithTLBackingMemory extends Config((site, here, up) => {
   case ExtTLMem => up(ExtMem, site) // enable TL backing memory
 })
 
-class WithTileFrequency(fMHz: Double) extends ClockNameContainsAssignment("core", fMHz)
+class WithSerialTLBackingMemory extends Config((site, here, up) => {
+  case ExtMem => None
+  case SerialTLKey => up(SerialTLKey, site).map { k => k.copy(
+    memParams = {
+      val memPortParams = up(ExtMem, site).get
+      require(memPortParams.nMemoryChannels == 1)
+      memPortParams.master
+    },
+    isMemoryDevice = true
+  )}
+})
+
+/**
+  * Mixins to define either a specific tile frequency for a single hart or all harts
+  *
+  * @param fMHz Frequency in MHz of the tile or all tiles
+  * @param hartId Optional hartid to assign the frequency to (if unspecified default to all harts)
+  */
+class WithTileFrequency(fMHz: Double, hartId: Option[Int] = None) extends ClockNameContainsAssignment({
+    hartId match {
+      case Some(id) => s"tile_$id"
+      case None => "tile"
+    }
+  },
+  fMHz)
 
 class WithPeripheryBusFrequencyAsDefault extends Config((site, here, up) => {
   case DefaultClockFrequencyKey => (site(PeripheryBusKey).dtsFrequency.get / (1000 * 1000)).toDouble

generators/chipyard/src/main/scala/DigitalTop.scala

@@ -13,6 +13,8 @@ import freechips.rocketchip.devices.tilelink._
 
 // DOC include start: DigitalTop
 class DigitalTop(implicit p: Parameters) extends ChipyardSystem
+  with testchipip.CanHavePeripheryCustomBootPin // Enables optional custom boot pin
+  with testchipip.HasPeripheryBootAddrReg // Use programmable boot address register
   with testchipip.CanHaveTraceIO // Enables optionally adding trace IO
   with testchipip.CanHaveBackingScratchpad // Enables optionally adding a backing scratchpad
   with testchipip.CanHavePeripheryBlockDevice // Enables optionally adding the block device

generators/chipyard/src/main/scala/HarnessBinders.scala

@@ -1,7 +1,8 @@
 package chipyard.harness
 
 import chisel3._
-import chisel3.experimental.{Analog, BaseModule}
+import chisel3.util._
+import chisel3.experimental.{Analog, BaseModule, DataMirror, Direction}
 
 import freechips.rocketchip.config.{Field, Config, Parameters}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike}
@@ -10,6 +11,7 @@ import freechips.rocketchip.devices.debug._
 import freechips.rocketchip.jtag.{JTAGIO}
 import freechips.rocketchip.system.{SimAXIMem}
 import freechips.rocketchip.subsystem._
+import freechips.rocketchip.util._
 
 import sifive.blocks.devices.gpio._
 import sifive.blocks.devices.uart._
@@ -19,8 +21,8 @@ import barstools.iocell.chisel._
 
 import testchipip._
 
-import chipyard.HasHarnessSignalReferences
-import chipyard.iobinders.GetSystemParameters
+import chipyard.{HasHarnessSignalReferences, HarnessClockInstantiatorKey}
+import chipyard.iobinders.{GetSystemParameters, JTAGChipIO}
 
 import tracegen.{TraceGenSystemModuleImp}
 import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
@@ -88,21 +90,21 @@ class WithUARTAdapter extends OverrideHarnessBinder({
 
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideHarnessBinder({
   (system: HasPeripherySPIFlashModuleImp, th: HasHarnessSignalReferences, ports: Seq[SPIChipIO]) => {
-    SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p)
+    SimSPIFlashModel.connect(ports, th.buildtopReset, rdOnly)(system.p)
   }
 })
 
 class WithSimBlockDevice extends OverrideHarnessBinder({
   (system: CanHavePeripheryBlockDevice, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    ports.map { b => SimBlockDevice.connect(b.clock, th.harnessReset.asBool, Some(b.bits)) }
+    ports.map { b => SimBlockDevice.connect(b.clock, th.buildtopReset.asBool, Some(b.bits)) }
   }
 })
 
 class WithBlockDeviceModel extends OverrideHarnessBinder({
   (system: CanHavePeripheryBlockDevice, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    ports.map { b => withClockAndReset(b.clock, th.harnessReset) { BlockDeviceModel.connect(Some(b.bits)) } }
+    ports.map { b => withClockAndReset(b.clock, th.buildtopReset) { BlockDeviceModel.connect(Some(b.bits)) } }
   }
 })
 
@@ -110,7 +112,7 @@ class WithLoopbackNIC extends OverrideHarnessBinder({
   (system: CanHavePeripheryIceNIC, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
     ports.map { n =>
-      withClockAndReset(n.clock, th.harnessReset) {
+      withClockAndReset(n.clock, th.buildtopReset) {
         NicLoopback.connect(Some(n.bits), p(NICKey))
       }
     }
@@ -120,7 +122,7 @@ class WithLoopbackNIC extends OverrideHarnessBinder({
 class WithSimNetwork extends OverrideHarnessBinder({
   (system: CanHavePeripheryIceNIC, th: BaseModule with HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    ports.map { n => SimNetwork.connect(Some(n.bits), n.clock, th.harnessReset.asBool) }
+    ports.map { n => SimNetwork.connect(Some(n.bits), n.clock, th.buildtopReset.asBool) }
   }
 })
 
@@ -137,14 +139,73 @@ class WithSimAXIMem extends OverrideHarnessBinder({
   }
 })
 
-class WithBlackBoxSimMem extends OverrideHarnessBinder({
+class WithSimAXIMemOverSerialTL extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    implicit val p = chipyard.iobinders.GetSystemParameters(system)
+
+    p(SerialTLKey).map({ sVal =>
+      require(sVal.axiMemOverSerialTLParams.isDefined)
+      val axiDomainParams = sVal.axiMemOverSerialTLParams.get
+      require(sVal.isMemoryDevice)
+
+      val memFreq = axiDomainParams.getMemFrequency(system.asInstanceOf[HasTileLinkLocations])
+
+      ports.map({ port =>
+// DOC include start: HarnessClockInstantiatorEx
+        withClockAndReset(th.buildtopClock, th.buildtopReset) {
+          val memOverSerialTLClockBundle = p(HarnessClockInstantiatorKey).requestClockBundle("mem_over_serial_tl_clock", memFreq)
+          val serial_bits = SerialAdapter.asyncQueue(port, th.buildtopClock, th.buildtopReset)
+          val harnessMultiClockAXIRAM = SerialAdapter.connectHarnessMultiClockAXIRAM(
+            system.serdesser.get,
+            serial_bits,
+            memOverSerialTLClockBundle,
+            th.buildtopReset)
+// DOC include end: HarnessClockInstantiatorEx
+          val success = SerialAdapter.connectSimSerial(harnessMultiClockAXIRAM.module.io.tsi_ser, th.buildtopClock, th.buildtopReset.asBool)
+          when (success) { th.success := true.B }
+
+          // connect SimDRAM from the AXI port coming from the harness multi clock axi ram
+          (harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi_port, edge) =>
+            val memSize = sVal.memParams.size
+            val lineSize = p(CacheBlockBytes)
+            val mem = Module(new SimDRAM(memSize, lineSize, BigInt(memFreq.toLong), edge.bundle)).suggestName("simdram")
+            mem.io.axi <> axi_port.bits
+            mem.io.clock := axi_port.clock
+            mem.io.reset := axi_port.reset
+          }
+        }
+      })
+    })
+  }
+})
+
+class WithBlackBoxSimMem(additionalLatency: Int = 0) extends OverrideHarnessBinder({
   (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[ClockedAndResetIO[AXI4Bundle]]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
     (ports zip system.memAXI4Node.edges.in).map { case (port, edge) =>
       val memSize = p(ExtMem).get.master.size
       val lineSize = p(CacheBlockBytes)
-      val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle)).suggestName("simdram")
+      val clockFreq = p(MemoryBusKey).dtsFrequency.get
+      val mem = Module(new SimDRAM(memSize, lineSize, clockFreq, edge.bundle)).suggestName("simdram")
       mem.io.axi <> port.bits
+      // Bug in Chisel implementation. See https://github.com/chipsalliance/chisel3/pull/1781
+      def Decoupled[T <: Data](irr: IrrevocableIO[T]): DecoupledIO[T] = {
+        require(DataMirror.directionOf(irr.bits) == Direction.Output, "Only safe to cast produced Irrevocable bits to Decoupled.")
+        val d = Wire(new DecoupledIO(chiselTypeOf(irr.bits)))
+        d.bits := irr.bits
+        d.valid := irr.valid
+        irr.ready := d.ready
+        d
+      }
+      if (additionalLatency > 0) {
+        withClockAndReset (port.clock, port.reset) {
+          mem.io.axi.aw <> (0 until additionalLatency).foldLeft(Decoupled(port.bits.aw))((t, _) => Queue(t, 1, pipe=true))
+          mem.io.axi.w  <> (0 until additionalLatency).foldLeft(Decoupled(port.bits.w ))((t, _) => Queue(t, 1, pipe=true))
+          port.bits.b   <> (0 until additionalLatency).foldLeft(Decoupled(mem.io.axi.b))((t, _) => Queue(t, 1, pipe=true))
+          mem.io.axi.ar <> (0 until additionalLatency).foldLeft(Decoupled(port.bits.ar))((t, _) => Queue(t, 1, pipe=true))
+          port.bits.r   <> (0 until additionalLatency).foldLeft(Decoupled(mem.io.axi.r))((t, _) => Queue(t, 1, pipe=true))
+        }
+      }
       mem.io.clock := port.clock
       mem.io.reset := port.reset
     }
@@ -183,11 +244,17 @@ class WithSimDebug extends OverrideHarnessBinder({
       case d: ClockedDMIIO =>
         val dtm_success = WireInit(false.B)
         when (dtm_success) { th.success := true.B }
-        val dtm = Module(new SimDTM).connect(th.harnessClock, th.harnessReset.asBool, d, dtm_success)
-      case j: JTAGIO =>
+        val dtm = Module(new SimDTM).connect(th.buildtopClock, th.buildtopReset.asBool, d, dtm_success)
+      case j: JTAGChipIO =>
         val dtm_success = WireInit(false.B)
         when (dtm_success) { th.success := true.B }
-        val jtag = Module(new SimJTAG(tickDelay=3)).connect(j, th.harnessClock, th.harnessReset.asBool, ~(th.harnessReset.asBool), dtm_success)
+        val jtag_wire = Wire(new JTAGIO)
+        jtag_wire.TDO.data := j.TDO
+        jtag_wire.TDO.driven := true.B
+        j.TCK := jtag_wire.TCK
+        j.TMS := jtag_wire.TMS
+        j.TDI := jtag_wire.TDI
+        val jtag = Module(new SimJTAG(tickDelay=3)).connect(jtag_wire, th.buildtopClock, th.buildtopReset.asBool, ~(th.buildtopReset.asBool), dtm_success)
     }
   }
 })
@@ -195,11 +262,10 @@ class WithSimDebug extends OverrideHarnessBinder({
 class WithTiedOffDebug extends OverrideHarnessBinder({
   (system: HasPeripheryDebug, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     ports.map {
-      case j: JTAGIO =>
+      case j: JTAGChipIO =>
         j.TCK := true.B.asClock
         j.TMS := true.B
         j.TDI := true.B
-        j.TRSTn.foreach { r => r := true.B }
       case d: ClockedDMIIO =>
         d.dmi.req.valid := false.B
         d.dmi.req.bits  := DontCare
@@ -221,8 +287,11 @@ class WithSerialAdapterTiedOff extends OverrideHarnessBinder({
   (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     implicit val p = chipyard.iobinders.GetSystemParameters(system)
     ports.map({ port =>
-      val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, port, th.harnessReset)
-      SerialAdapter.tieoff(ram.module.io.tsi_ser)
+      val bits = SerialAdapter.asyncQueue(port, th.buildtopClock, th.buildtopReset)
+      withClockAndReset(th.buildtopClock, th.buildtopReset) {
+        val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, bits, th.buildtopReset)
+        SerialAdapter.tieoff(ram.module.io.tsi_ser)
+      }
     })
   }
 })
@@ -231,9 +300,12 @@ class WithSimSerial extends OverrideHarnessBinder({
   (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     implicit val p = chipyard.iobinders.GetSystemParameters(system)
     ports.map({ port =>
-      val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, port, th.harnessReset)
-      val success = SerialAdapter.connectSimSerial(ram.module.io.tsi_ser, port.clock, th.harnessReset.asBool)
-      when (success) { th.success := true.B }
+      val bits = SerialAdapter.asyncQueue(port, th.buildtopClock, th.buildtopReset)
+      withClockAndReset(th.buildtopClock, th.buildtopReset) {
+        val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, bits, th.buildtopReset)
+        val success = SerialAdapter.connectSimSerial(ram.module.io.tsi_ser, th.buildtopClock, th.buildtopReset.asBool)
+        when (success) { th.success := true.B }
+      }
     })
   }
 })
@@ -249,3 +321,9 @@ class WithSimDromajoBridge extends ComposeHarnessBinder({
     ports.map { p => p.traces.map(tileTrace => SimDromajoBridge(tileTrace)(system.p)) }
   }
 })
+
+class WithTieOffCustomBootPin extends OverrideHarnessBinder({
+  (system: CanHavePeripheryCustomBootPin, th: HasHarnessSignalReferences, ports: Seq[Bool]) => {
+    ports.foreach(_ := false.B)
+  }
+})

generators/chipyard/src/main/scala/IOBinders.scala

@@ -199,6 +199,13 @@ class WithExtInterruptIOCells extends OverrideIOBinder({
   }
 })
 
+// Rocketchip's JTAGIO exposes the oe signal, which doesn't go off-chip
+class JTAGChipIO extends Bundle {
+  val TCK = Input(Clock())
+  val TMS = Input(Bool())
+  val TDI = Input(Bool())
+  val TDO = Output(Bool())
+}
 
 class WithDebugIOCells extends OverrideLazyIOBinder({
   (system: HasPeripheryDebug) => {
@@ -238,7 +245,12 @@ class WithDebugIOCells extends OverrideLazyIOBinder({
         }
 
         val jtagTuple = debug.systemjtag.map { j =>
-          IOCell.generateIOFromSignal(j.jtag, "jtag", p(IOCellKey), abstractResetAsAsync = true)
+          val jtag_wire = Wire(new JTAGChipIO)
+          j.jtag.TCK := jtag_wire.TCK
+          j.jtag.TMS := jtag_wire.TMS
+          j.jtag.TDI := jtag_wire.TDI
+          jtag_wire.TDO := j.jtag.TDO.data
+          IOCell.generateIOFromSignal(jtag_wire, "jtag", p(IOCellKey), abstractResetAsAsync = true)
         }
 
         val apbTuple = debug.apb.map { a =>
@@ -260,7 +272,6 @@ class WithSerialTLIOCells extends OverrideIOBinder({
   }).getOrElse((Nil, Nil))
 })
 
-
 class WithAXI4MemPunchthrough extends OverrideLazyIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
@@ -361,6 +372,13 @@ class WithTraceIOPunchthrough extends OverrideIOBinder({
   }
 })
 
+class WithCustomBootPin extends OverrideIOBinder({
+  (system: CanHavePeripheryCustomBootPin) => system.custom_boot_pin.map({ p =>
+    val sys = system.asInstanceOf[BaseSubsystem]
+    val (port, cells) = IOCell.generateIOFromSignal(p.getWrappedValue, "custom_boot", sys.p(IOCellKey), abstractResetAsAsync = true)
+    (Seq(port), cells)
+  }).getOrElse((Nil, Nil))
+})
 
 class WithDontTouchPorts extends OverrideIOBinder({
   (system: DontTouch) => system.dontTouchPorts(); (Nil, Nil)

generators/chipyard/src/main/scala/TestHarness.scala

@@ -1,12 +1,16 @@
 package chipyard
 
 import chisel3._
-import scala.collection.mutable.{ArrayBuffer}
+
+import scala.collection.mutable.{ArrayBuffer, LinkedHashMap}
 import freechips.rocketchip.diplomacy.{LazyModule}
 import freechips.rocketchip.config.{Field, Parameters}
+import freechips.rocketchip.util.{ResetCatchAndSync}
+import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters, ClockSinkParameters, ClockParameters}
 
 import chipyard.harness.{ApplyHarnessBinders, HarnessBinders}
 import chipyard.iobinders.HasIOBinders
+import chipyard.clocking.{SimplePllConfiguration, ClockDividerN}
 
 // -------------------------------
 // Chipyard Test Harness
@@ -19,26 +23,84 @@ trait HasTestHarnessFunctions {
 }
 
 trait HasHarnessSignalReferences {
-  def harnessClock: Clock
-  def harnessReset: Reset
+  // clock/reset of the chiptop reference clock (can be different than the implicit harness clock/reset)
+  def buildtopClock: Clock
+  def buildtopReset: Reset
   def dutReset: Reset
   def success: Bool
 }
 
+class HarnessClockInstantiator {
+  private val _clockMap: LinkedHashMap[String, (Double, ClockBundle)] = LinkedHashMap.empty
+
+  // request a clock bundle at a particular frequency
+  def requestClockBundle(name: String, freqRequested: Double): ClockBundle = {
+    val clockBundle = Wire(new ClockBundle(ClockBundleParameters()))
+    _clockMap(name) = (freqRequested, clockBundle)
+    clockBundle
+  }
+
+  // connect all clock wires specified to a divider only PLL
+  def instantiateHarnessDividerPLL(refClock: ClockBundle): Unit = {
+    val sinks = _clockMap.map({ case (name, (freq, bundle)) =>
+      ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq / (1000 * 1000))), name=Some(name))
+    }).toSeq
+
+    val pllConfig = new SimplePllConfiguration("harnessDividerOnlyClockGenerator", sinks)
+    pllConfig.emitSummaries()
+
+    val dividedClocks = LinkedHashMap[Int, Clock]()
+    def instantiateDivider(div: Int): Clock = {
+      val divider = Module(new ClockDividerN(div))
+      divider.suggestName(s"ClockDivideBy${div}")
+      divider.io.clk_in := refClock.clock
+      dividedClocks(div) = divider.io.clk_out
+      divider.io.clk_out
+    }
+
+    // connect wires to clock source
+    for (sinkParams <- sinks) {
+      // bypass the reference freq. (don't create a divider + reset sync)
+      val (divClock, divReset) = if (sinkParams.take.get.freqMHz != pllConfig.referenceFreqMHz) {
+        val div = pllConfig.sinkDividerMap(sinkParams)
+        val divClock = dividedClocks.getOrElse(div, instantiateDivider(div))
+        (divClock, ResetCatchAndSync(divClock, refClock.reset.asBool))
+      } else {
+        (refClock.clock, refClock.reset)
+      }
+
+      _clockMap(sinkParams.name.get)._2.clock := divClock
+      _clockMap(sinkParams.name.get)._2.reset := divReset
+    }
+  }
+}
+
+case object HarnessClockInstantiatorKey extends Field[HarnessClockInstantiator](new HarnessClockInstantiator)
+
 class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSignalReferences {
   val io = IO(new Bundle {
     val success = Output(Bool())
   })
 
+  val buildtopClock = Wire(Clock())
+  val buildtopReset = Wire(Reset())
+
   val lazyDut = LazyModule(p(BuildTop)(p)).suggestName("chiptop")
   val dut = Module(lazyDut.module)
+
   io.success := false.B
 
-  val harnessClock = clock
-  val harnessReset = WireInit(reset)
-  val success = io.success
+  val freqMHz = lazyDut match {
+    case d: HasReferenceClockFreq => d.refClockFreqMHz
+    case _ => p(DefaultClockFrequencyKey)
+  }
+  val refClkBundle = p(HarnessClockInstantiatorKey).requestClockBundle("buildtop_reference_clock", freqMHz * (1000 * 1000))
 
-  val dutReset = reset.asAsyncReset
+  buildtopClock := refClkBundle.clock
+  buildtopReset := WireInit(refClkBundle.reset)
+  val dutReset = refClkBundle.reset.asAsyncReset
+
+  val success = io.success
 
   lazyDut match { case d: HasTestHarnessFunctions =>
     d.harnessFunctions.foreach(_(this))
@@ -46,5 +108,10 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
   lazyDut match { case d: HasIOBinders =>
     ApplyHarnessBinders(this, d.lazySystem, d.portMap)
   }
+
+  val implicitHarnessClockBundle = Wire(new ClockBundle(ClockBundleParameters()))
+  implicitHarnessClockBundle.clock := clock
+  implicitHarnessClockBundle.reset := reset
+  p(HarnessClockInstantiatorKey).instantiateHarnessDividerPLL(implicitHarnessClockBundle)
 }
 

generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala

@@ -51,7 +51,7 @@ object FrequencyUtils {
     require(!requestedOutputs.contains(0.0))
     val requestedFreqs = requestedOutputs.map(_.freqMHz)
     val fastestFreq = requestedFreqs.max
-    require(fastestFreq < maximumAllowableFreqMHz)
+    require(fastestFreq <= maximumAllowableFreqMHz)
 
     val candidateFreqs =
       Seq.tabulate(Math.ceil(maximumAllowableFreqMHz / fastestFreq).toInt)(i => (i + 1) * fastestFreq)
@@ -89,6 +89,7 @@ class SimplePllConfiguration(
      ElaborationArtefacts.add(s"${name}.freq-summary", summaryString)
      println(summaryString)
    }
+   def referenceSinkParams(): ClockSinkParameters = sinkDividerMap.find(_._2 == 1).get._1
 }
 
 case class DividerOnlyClockGeneratorNode(pllName: String)(implicit valName: ValName)
@@ -144,7 +145,3 @@ class DividerOnlyClockGenerator(pllName: String)(implicit p: Parameters, valName
     }
   }
 }
-
-object DividerOnlyClockGenerator {
-  def apply()(implicit p: Parameters, valName: ValName) = LazyModule(new DividerOnlyClockGenerator(valName.name)).node
-}

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -21,6 +21,7 @@ class AbstractConfig extends Config(
   new chipyard.harness.WithSimAXIMMIO ++                        // add SimAXIMem for axi4 mmio port, if enabled
   new chipyard.harness.WithTieOffInterrupts ++                  // tie-off interrupt ports, if present
   new chipyard.harness.WithTieOffL2FBusAXI ++                   // tie-off external AXI4 master, if present
+  new chipyard.harness.WithTieOffCustomBootPin ++
 
   // The IOBinders instantiate ChipTop IOs to match desired digital IOs
   // IOCells are generated for "Chip-like" IOs, while simulation-only IOs are directly punched through
@@ -37,6 +38,7 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithSPIIOCells ++
   new chipyard.iobinders.WithTraceIOPunchthrough ++
   new chipyard.iobinders.WithExtInterruptIOCells ++
+  new chipyard.iobinders.WithCustomBootPin ++
 
   new testchipip.WithDefaultSerialTL ++                          // use serialized tilelink port to external serialadapter/harnessRAM
   new chipyard.config.WithBootROM ++                             // use default bootrom
@@ -45,6 +47,8 @@ class AbstractConfig extends Config(
   new chipyard.config.WithNoSubsystemDrivenClocks ++             // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
   new chipyard.config.WithInheritBusFrequencyAssignments ++      // Unspecified clocks within a bus will receive the bus frequency if set
   new chipyard.config.WithPeripheryBusFrequencyAsDefault ++      // Unspecified frequencies with match the pbus frequency (which is always set)
+  new chipyard.config.WithMemoryBusFrequency(100.0) ++           // Default 100 MHz mbus
+  new chipyard.config.WithPeripheryBusFrequency(100.0) ++        // Default 100 MHz pbus
   new freechips.rocketchip.subsystem.WithJtagDTM ++              // set the debug module to expose a JTAG port
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++           // no top-level MMIO master port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithNoSlavePort ++          // no top-level MMIO slave port (overrides default set in rocketchip)
@@ -52,4 +56,3 @@ class AbstractConfig extends Config(
   new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++ // no external interrupts
   new chipyard.WithMulticlockCoherentBusTopology ++              // hierarchical buses including mbus+l2
   new freechips.rocketchip.system.BaseConfig)                    // "base" rocketchip system
-

generators/chipyard/src/main/scala/config/HeteroConfigs.scala

@@ -20,23 +20,35 @@ class HwachaLargeBoomAndHwachaRocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 // DOC include end: BoomAndRocketWithHwacha
 
-// DOC include start: DualBoomAndRocketOneHwacha
 class LargeBoomAndHwachaRocketConfig extends Config(
   new chipyard.config.WithMultiRoCC ++                                  // support heterogeneous rocc
-  new chipyard.config.WithMultiRoCCHwacha(1) ++                         // put hwacha on hart-1 (rocket)
+  new chipyard.config.WithMultiRoCCHwacha(0) ++                         // put hwacha on hart-0 (rocket)
   new hwacha.DefaultHwachaConfig ++                                     // set default hwacha config keys
   new boom.common.WithNLargeBooms(1) ++                                 // add 1 boom core
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++                // add 1 rocket core
   new chipyard.config.AbstractConfig)
+
+// DOC include start: DualBoomAndRocketOneHwacha
+class DualLargeBoomAndHwachaRocketConfig extends Config(
+  new chipyard.config.WithMultiRoCC ++                                  // support heterogeneous rocc
+  new chipyard.config.WithMultiRoCCHwacha(0) ++                         // put hwacha on hart-0 (rocket)
+  new hwacha.DefaultHwachaConfig ++                                     // set default hwacha config keys
+  new boom.common.WithNLargeBooms(2) ++                                 // add 2 boom cores
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++                // add 1 rocket core
+  new chipyard.config.AbstractConfig)
 // DOC include end: DualBoomAndRocketOneHwacha
 
-
-// DOC include start: DualBoomAndRocket
 class DualLargeBoomAndDualRocketConfig extends Config(
   new boom.common.WithNLargeBooms(2) ++                   // add 2 boom cores
   new freechips.rocketchip.subsystem.WithNBigCores(2) ++  // add 2 rocket cores
   new chipyard.config.AbstractConfig)
-// DOC include end: DualBoomAndRocket
+
+// DOC include start: DualBoomAndSingleRocket
+class DualLargeBoomAndSingleRocketConfig extends Config(
+  new boom.common.WithNLargeBooms(2) ++                   // add 2 boom cores
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++  // add 1 rocket core
+  new chipyard.config.AbstractConfig)
+// DOC include end: DualBoomAndSingleRocket
 
 class LargeBoomAndRocketWithControlCoreConfig extends Config(
   new freechips.rocketchip.subsystem.WithNSmallCores(1) ++ // Add a small "control" core

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -34,6 +34,12 @@ class GemminiRocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 // DOC include end: GemminiRocketConfig
 
+class FPGemminiRocketConfig extends Config(
+  new gemmini.GemminiFP32DefaultConfig ++                         // use FP32Gemmini systolic array GEMM accelerator
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
+
+
 // DOC include start: DmiRocket
 class dmiRocketConfig extends Config(
   new chipyard.harness.WithSerialAdapterTiedOff ++               // don't attach an external SimSerial
@@ -206,3 +212,26 @@ class LBWIFRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNoMemPort ++       // remove AXI4 backing memory
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
+
+// DOC include start: MulticlockAXIOverSerialConfig
+class MulticlockAXIOverSerialConfig extends Config(
+  new chipyard.config.WithSystemBusFrequencyAsDefault ++
+  new chipyard.config.WithSystemBusFrequency(250) ++
+  new chipyard.config.WithPeripheryBusFrequency(250) ++
+  new chipyard.config.WithMemoryBusFrequency(250) ++
+  new chipyard.config.WithFrontBusFrequency(50) ++
+  new chipyard.config.WithTileFrequency(500, Some(1)) ++
+  new chipyard.config.WithTileFrequency(250, Some(0)) ++
+
+  new chipyard.config.WithFbusToSbusCrossingType(AsynchronousCrossing()) ++
+  new testchipip.WithAsynchronousSerialSlaveCrossing ++
+  new freechips.rocketchip.subsystem.WithAsynchronousRocketTiles(
+    AsynchronousCrossing().depth,
+    AsynchronousCrossing().sourceSync) ++
+
+  new chipyard.harness.WithSimAXIMemOverSerialTL ++ // add SimDRAM DRAM model for axi4 backing memory over the SerDes link, if axi4 mem is enabled
+  new chipyard.config.WithSerialTLBackingMemory ++ // remove axi4 mem port in favor of SerialTL memory
+
+  new freechips.rocketchip.subsystem.WithNBigCores(2) ++
+  new chipyard.config.AbstractConfig)
+// DOC include end: MulticlockAXIOverSerialConfig

generators/chipyard/src/main/scala/config/TracegenConfigs.scala

@@ -11,6 +11,8 @@ class AbstractTraceGenConfig extends Config(
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new chipyard.config.WithPeripheryBusFrequencyAsDefault ++
+  new chipyard.config.WithMemoryBusFrequency(100.0) ++
+  new chipyard.config.WithPeripheryBusFrequency(100.0) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
 

generators/firechip/src/main/scala/BridgeBinders.scala

@@ -12,6 +12,8 @@ import freechips.rocketchip.devices.debug.{Debug, HasPeripheryDebugModuleImp}
 import freechips.rocketchip.amba.axi4.{AXI4Bundle}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tile.{RocketTile}
+import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters}
+import freechips.rocketchip.util.{ResetCatchAndSync}
 import sifive.blocks.devices.uart._
 
 import testchipip._
@@ -70,8 +72,11 @@ class WithSerialBridge extends OverrideHarnessBinder({
   (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[ClockedIO[SerialIO]]) => {
     ports.map { port =>
       implicit val p = GetSystemParameters(system)
-      val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, port, th.harnessReset)
-      SerialBridge(port.clock, ram.module.io.tsi_ser, p(ExtMem).map(_ => MainMemoryConsts.globalName))
+      val bits = SerialAdapter.asyncQueue(port, th.buildtopClock, th.buildtopReset)
+      val ram = withClockAndReset(th.buildtopClock, th.buildtopReset) {
+        SerialAdapter.connectHarnessRAM(system.serdesser.get, bits, th.buildtopReset)
+      }
+      SerialBridge(th.buildtopClock, ram.module.io.tsi_ser, p(ExtMem).map(_ => MainMemoryConsts.globalName))
     }
     Nil
   }
@@ -98,7 +103,56 @@ class WithUARTBridge extends OverrideHarnessBinder({
 class WithBlockDeviceBridge extends OverrideHarnessBinder({
   (system: CanHavePeripheryBlockDevice, th: FireSim, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessReset.toBool) }
+    ports.map { b => BlockDevBridge(b.clock, b.bits, th.buildtopReset.asBool) }
+    Nil
+  }
+})
+
+class WithAXIOverSerialTLCombinedBridges extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[ClockedIO[SerialIO]]) => {
+    implicit val p = GetSystemParameters(system)
+
+    p(SerialTLKey).map({ sVal =>
+      require(sVal.axiMemOverSerialTLParams.isDefined)
+      val axiDomainParams = sVal.axiMemOverSerialTLParams.get
+      require(sVal.isMemoryDevice)
+
+      val memFreq = axiDomainParams.getMemFrequency(system.asInstanceOf[HasTileLinkLocations])
+
+      ports.map({ port =>
+        val axiClock = p(ClockBridgeInstantiatorKey).requestClock("mem_over_serial_tl_clock", memFreq)
+        val axiClockBundle = Wire(new ClockBundle(ClockBundleParameters()))
+        axiClockBundle.clock := axiClock
+        axiClockBundle.reset := ResetCatchAndSync(axiClock, th.buildtopReset.asBool)
+
+        val serial_bits = SerialAdapter.asyncQueue(port, th.buildtopClock, th.buildtopReset)
+
+        val harnessMultiClockAXIRAM = withClockAndReset(th.buildtopClock, th.buildtopReset) {
+          SerialAdapter.connectHarnessMultiClockAXIRAM(
+            system.serdesser.get,
+            serial_bits,
+            axiClockBundle,
+            th.buildtopReset)
+        }
+        SerialBridge(th.buildtopClock, harnessMultiClockAXIRAM.module.io.tsi_ser, Some(MainMemoryConsts.globalName))
+
+        // connect SimAxiMem
+        (harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi4, edge) =>
+          val nastiKey = NastiParameters(axi4.bits.r.bits.data.getWidth,
+                                        axi4.bits.ar.bits.addr.getWidth,
+                                        axi4.bits.ar.bits.id.getWidth)
+          system match {
+            case s: BaseSubsystem => FASEDBridge(axi4.clock, axi4.bits, axi4.reset.asBool,
+              CompleteConfig(p(firesim.configs.MemModelKey),
+                            nastiKey,
+                            Some(AXI4EdgeSummary(edge)),
+                            Some(MainMemoryConsts.globalName)))
+            case _ => throw new Exception("Attempting to attach FASED Bridge to misconfigured design")
+          }
+        }
+      })
+    })
+
     Nil
   }
 })
@@ -138,7 +192,7 @@ class WithDromajoBridge extends ComposeHarnessBinder({
 
 class WithTraceGenBridge extends OverrideHarnessBinder({
   (system: TraceGenSystemModuleImp, th: FireSim, ports: Seq[Bool]) =>
-    ports.map { p => GroundTestBridge(th.harnessClock, p)(system.p) }; Nil
+    ports.map { p => GroundTestBridge(th.buildtopClock, p)(system.p) }; Nil
 })
 
 class WithFireSimMultiCycleRegfile extends ComposeIOBinder({
@@ -170,6 +224,7 @@ class WithFireSimFAME5 extends ComposeIOBinder({
         annotate(EnableModelMultiThreadingAnnotation(b.module))
       case r: RocketTile =>
         annotate(EnableModelMultiThreadingAnnotation(r.module))
+      case _ => Nil
     }
     (Nil, Nil)
   }

generators/firechip/src/main/scala/FireSim.scala

@@ -2,6 +2,8 @@
 
 package firesim.firesim
 
+import scala.collection.mutable.{LinkedHashMap}
+
 import chisel3._
 import chisel3.experimental.{IO}
 
@@ -38,44 +40,113 @@ object NodeIdx {
 /**
   * Under FireSim's current multiclock implementation there can be only a
   * single clock bridge. This requires, therefore, that it  be instantiated in
-  * the harness and reused across all supernode instances. This class attempts to 
+  * the harness and reused across all supernode instances. This class attempts to
   * memoize its instantiation such that it can be referenced from within a ClockScheme function.
   */
 class ClockBridgeInstantiator {
-  private var _clockRecord: Option[RecordMap[Clock]] = None
+  private val _harnessClockMap: LinkedHashMap[String, (Double, Clock)] = LinkedHashMap.empty
 
-  def getClockRecord: RecordMap[Clock] = _clockRecord.get
+  // Assumes that the supernode implementation results in duplicated clocks
+  //   (i.e. only 1 set of clocks is generated for all BuildTop designs)
+  private val _buildtopClockMap: LinkedHashMap[String, (RationalClock, Clock)] = LinkedHashMap.empty
+  private var _buildtopRefTuple: Option[(String, Double)] = None
 
-  def getClockRecordOrInstantiate(allClocks: Seq[RationalClock], baseClockName: String): RecordMap[Clock] = {
-    if (_clockRecord.isEmpty) {
-      require(allClocks.exists(_.name == baseClockName),
-        s"Provided base-clock name, ${baseClockName}, does not match a defined clock. Available clocks:\n " +
-        allClocks.map(_.name).mkString("\n "))
+  /**
+   * Request a clock at a particular frequency
+   *
+   * @param name An identifier for the associated clock domain
+   *
+   * @param freqRequested Freq. for the domain in Hz
+   */
+  def requestClock(name: String, freqRequested: Double): Clock = {
+    val clkWire = Wire(new Clock)
+    _harnessClockMap(name) = (freqRequested, clkWire)
+    clkWire
+  }
 
-      val baseClock = allClocks.find(_.name == baseClockName).get
-      val simplified = allClocks.map { c =>
-        c.copy(multiplier = c.multiplier * baseClock.divisor, divisor = c.divisor * baseClock.multiplier)
-         .simplify
-      }
+  /**
+   * Get a RecordMap of clocks for a set of input RationalClocks
+   *
+   * @param allClocks Seq. of RationalClocks that want a clock
+   *
+   * @param baseClockName Name of domain that the allClocks is rational to
+   *
+   * @param baseFreqRequested Freq. for the reference domain in Hz
+   */
+  def requestClockRecordMap(allClocks: Seq[RationalClock], baseClockName: String, baseFreqRequested: Double): RecordMap[Clock] = {
+    require(!_buildtopRefTuple.isDefined, "Can only request one RecordMap of Clocks")
 
-    /**
-      * Removes clocks that have the same frequency before instantiating the
-      * clock bridge to avoid unnecessary BUFGCE use.
-      */
-      val distinct = simplified.foldLeft(Seq(RationalClock(baseClockName, 1, 1))) { case (list, candidate) =>
-        if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate
-      }
+    val ratClockRecordMapWire = Wire(RecordMap(allClocks.map { c => (c.name, Clock()) }:_*))
 
-      val clockBridge = Module(new RationalClockBridge(distinct))
-      val cbVecTuples = distinct.zip(clockBridge.io.clocks)
-      val outputWire = Wire(RecordMap(simplified.map { c => (c.name, Clock()) }:_*))
-      for (parameter <- simplified) {
-        val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(parameter)).get
-        outputWire(parameter.name).get := cbClockField
-      }
-      _clockRecord = Some(outputWire)
+    _buildtopRefTuple = Some((baseClockName, baseFreqRequested))
+    for (clock <- allClocks) {
+      val clkWire = Wire(new Clock)
+      _buildtopClockMap(clock.name) = (clock, clkWire)
+      ratClockRecordMapWire(clock.name).get := clkWire
+    }
+
+    ratClockRecordMapWire
+  }
+
+  /**
+   * Connect all clocks requested to ClockBridge
+   */
+  def instantiateFireSimClockBridge: Unit = {
+    require(_buildtopRefTuple.isDefined, "Must have rational clocks to assign to")
+    require(_buildtopClockMap.exists(_._1 == _buildtopRefTuple.get._1),
+      s"Provided base-clock name for rational clocks, ${_buildtopRefTuple.get._1}, doesn't match a name within specified rational clocks." +
+      "Available clocks:\n " + _buildtopClockMap.map(_._1).mkString("\n "))
+
+    // Simplify the RationalClocks ratio's
+    val refRatClock = _buildtopClockMap.find(_._1 == _buildtopRefTuple.get._1).get._2._1
+    val simpleRatClocks = _buildtopClockMap.map { t =>
+      val ratClock = t._2._1
+      ratClock.copy(
+        multiplier = ratClock.multiplier * refRatClock.divisor,
+        divisor = ratClock.divisor * refRatClock.multiplier).simplify
+    }
+
+    // Determine all the clock dividers (harness + rational clocks)
+    //   Note: Requires that the BuildTop reference frequency is requested with proper freq.
+    val refRatClockFreq = _buildtopRefTuple.get._2
+    val refRatSinkParams = ClockSinkParameters(take=Some(ClockParameters(freqMHz=refRatClockFreq / (1000 * 1000))),name=Some(_buildtopRefTuple.get._1))
+    val harSinkParams = _harnessClockMap.map { case (name, (freq, bundle)) =>
+      ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq / (1000 * 1000))),name=Some(name))
+    }.toSeq
+    val allSinkParams = harSinkParams :+ refRatSinkParams
+
+    // Use PLL config to determine overall div's
+    val pllConfig = new SimplePllConfiguration("firesimOverallClockBridge", allSinkParams)
+    pllConfig.emitSummaries
+
+    // Adjust all BuildTop RationalClocks with the div determined by the PLL
+    val refRatDiv = pllConfig.sinkDividerMap(refRatSinkParams)
+    val adjRefRatClocks = simpleRatClocks.map { clock =>
+      clock.copy(divisor = clock.divisor * refRatDiv).simplify
+    }
+
+    // Convert harness clocks to RationalClocks
+    val harRatClocks = harSinkParams.map { case ClockSinkParameters(_, _, _, _, clkParamsOpt, nameOpt) =>
+      RationalClock(nameOpt.get, 1, pllConfig.referenceFreqMHz.toInt / clkParamsOpt.get.freqMHz.toInt)
+    }
+
+    val allAdjRatClks = adjRefRatClocks ++ harRatClocks
+
+    // Removes clocks that have the same frequency before instantiating the
+    //   clock bridge to avoid unnecessary BUFGCE use.
+    val allDistinctRatClocks = allAdjRatClks.foldLeft(Seq(RationalClock(pllConfig.referenceSinkParams.name.get, 1, 1))) {
+      case (list, candidate) => if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate
+    }
+
+    val clockBridge = Module(new RationalClockBridge(allDistinctRatClocks))
+    val cbVecTuples = allDistinctRatClocks.zip(clockBridge.io.clocks)
+
+    // Connect all clocks (harness + BuildTop clocks)
+    for (clock <- allAdjRatClks) {
+      val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(clock)).get
+      _buildtopClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField }
+      _harnessClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField }
     }
-    getClockRecord
   }
 }
 
@@ -117,29 +188,35 @@ class WithFireSimSimpleClocks extends Config((site, here, up) => {
         clockBundle.reset := reset
       }
 
-      val pllConfig = new SimplePllConfiguration("FireSim RationalClockBridge", clockGroupEdge.sink.members)
+      val pllConfig = new SimplePllConfiguration("firesimBuildTopClockGenerator", clockGroupEdge.sink.members)
       pllConfig.emitSummaries
       val rationalClockSpecs = for ((sinkP, division) <- pllConfig.sinkDividerMap) yield {
         RationalClock(sinkP.name.get, 1, division)
       }
 
       chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
-        reset := th.harnessReset
+        reset := th.buildtopReset
         input_clocks := p(ClockBridgeInstantiatorKey)
-          .getClockRecordOrInstantiate(rationalClockSpecs.toSeq, p(FireSimBaseClockNameKey))
+          .requestClockRecordMap(rationalClockSpecs.toSeq, p(FireSimBaseClockNameKey), pllConfig.referenceFreqMHz * (1000 * 1000))
         Nil })
+
+      // return the reference frequency
+      pllConfig.referenceFreqMHz
     }
   }
 })
 
 class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSignalReferences {
   freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
-  val harnessClock = Wire(Clock())
-  val harnessReset = WireInit(false.B)
-  val peekPokeBridge = PeekPokeBridge(harnessClock, harnessReset)
+
+  val buildtopClock = Wire(Clock())
+  val buildtopReset = WireInit(false.B)
+  val peekPokeBridge = PeekPokeBridge(buildtopClock, buildtopReset)
   def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B }
   def success = { require(false, "success should not be used in Firesim"); false.B }
 
+  var btFreqMHz: Option[Double] = None
+
   // Instantiate multiple instances of the DUT to implement supernode
   for (i <- 0 until p(NumNodes)) {
     // It's not a RC bump without some hacks...
@@ -151,6 +228,12 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
       case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
     })))
     val module = Module(lazyModule.module)
+
+    btFreqMHz = Some(lazyModule match {
+      case d: HasReferenceClockFreq => d.refClockFreqMHz
+      case _ => p(DefaultClockFrequencyKey)
+    })
+
     lazyModule match { case d: HasTestHarnessFunctions =>
       require(d.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
       d.harnessFunctions.foreach(_(this))
@@ -160,5 +243,8 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
     }
     NodeIdx.increment()
   }
-  harnessClock := p(ClockBridgeInstantiatorKey).getClockRecord("implicit_clock").get
+
+  buildtopClock := p(ClockBridgeInstantiatorKey).requestClock("buildtop_reference_clock", btFreqMHz.get * (1000 * 1000))
+
+  p(ClockBridgeInstantiatorKey).instantiateFireSimClockBridge
 }

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -59,28 +59,14 @@ class WithNIC extends icenet.WithIceNIC(inBufFlits = 8192, ctrlQueueDepth = 64)
 class WithNVDLALarge extends nvidia.blocks.dla.WithNVDLA("large")
 class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small")
 
-
-// Tweaks that are generally applied to all firesim configs
-class WithFireSimConfigTweaks extends Config(
+// Non-frequency tweaks that are generally applied to all firesim configs
+class WithFireSimDesignTweaks extends Config(
   // Required: Bake in the default FASED memory model
   new WithDefaultMemModel ++
   // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
   new WithFireSimSimpleClocks ++
   // Required*: When using FireSim-as-top to provide a correct path to the target bootrom source
   new WithBootROM ++
-  // Optional: This sets the default frequency for all buses in the system to 3.2 GHz
-  // (since unspecified bus frequencies will use the pbus frequency)
-  // This frequency selection matches FireSim's legacy selection and is required
-  // to support 200Gb NIC performance. You may select a smaller value.
-  new chipyard.config.WithPeripheryBusFrequency(3200.0) ++
-  // Optional: These three configs put the DRAM memory system in it's own clock domian.
-  // Removing the first config will result in the FASED timing model running
-  // at the pbus freq (above, 3.2 GHz), which is outside the range of valid DDR3 speedgrades.
-  // 1 GHz matches the FASED default, using some other frequency will require
-  // runnings the FASED runtime configuration generator to generate faithful DDR3 timing values.
-  new chipyard.config.WithMemoryBusFrequency(1000.0) ++
-  new chipyard.config.WithAsynchrousMemoryBusCrossing ++
-  new testchipip.WithAsynchronousSerialSlaveCrossing ++
   // Required: Existing FAME-1 transform cannot handle black-box clock gates
   new WithoutClockGating ++
   // Required*: Removes thousands of assertions that would be synthesized (* pending PriorityMux bugfix)
@@ -89,10 +75,6 @@ class WithFireSimConfigTweaks extends Config(
   new chipyard.config.WithTraceIO ++
   // Optional: Request 16 GiB of target-DRAM by default (can safely request up to 32 GiB on F1)
   new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 16L) ++
-  // Required: Adds IO to attach SerialBridge. The SerialBridges is responsible
-  // for signalling simulation termination under simulation success. This fragment can
-  // be removed if you supply an auxiliary bridge that signals simulation termination
-  new testchipip.WithDefaultSerialTL ++
   // Optional: Removing this will require using an initramfs under linux
   new testchipip.WithBlockDevice ++
   // Optional: Set a UART baudrate (this selection matches FireSim's historical value)
@@ -101,6 +83,27 @@ class WithFireSimConfigTweaks extends Config(
   new chipyard.config.WithNoDebug
 )
 
+// Tweaks to modify target clock frequencies / crossings to firesim defaults
+class WithFireSimDefaultFrequencyTweaks extends Config(
+  // Optional*: Removing this will require adjusting the UART baud rate and
+  // potential target-software changes to properly capture UART output
+  new chipyard.config.WithPeripheryBusFrequency(3200.0) ++
+  // Optional: These three configs put the DRAM memory system in it's own clock domian.
+  // Removing the first config will result in the FASED timing model running
+  // at the pbus freq (above, 3.2 GHz), which is outside the range of valid DDR3 speedgrades.
+  // 1 GHz matches the FASED default, using some other frequency will require
+  // runnings the FASED runtime configuration generator to generate faithful DDR3 timing values.
+  new chipyard.config.WithMemoryBusFrequency(1000.0) ++
+  new chipyard.config.WithAsynchrousMemoryBusCrossing ++
+  new testchipip.WithAsynchronousSerialSlaveCrossing
+)
+
+// Tweaks that are generally applied to all firesim configs
+class WithFireSimConfigTweaks extends Config(
+  new WithFireSimDefaultFrequencyTweaks ++
+  new WithFireSimDesignTweaks
+)
+
 /*******************************************************************************
 * Full TARGET_CONFIG configurations. These set parameters of the target being
 * simulated.
@@ -206,6 +209,15 @@ class FireSimMulticlockRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithRationalRocketTiles ++   // Add rational crossings between RocketTile and uncore
   new FireSimRocketConfig)
 
+class FireSimMulticlockAXIOverSerialConfig extends Config(
+  new WithAXIOverSerialTLCombinedBridges ++ // use combined bridge to connect to axi mem over serial
+  new WithDefaultFireSimBridges ++
+  new testchipip.WithBlockDevice(false) ++ // disable blockdev
+  new WithDefaultMemModel ++
+  new WithFireSimDesignTweaks ++ // don't inherit firesim clocking
+  new chipyard.MulticlockAXIOverSerialConfig
+)
+
 //**********************************************************************************
 // System with 16 LargeBOOMs that can be simulated with Golden Gate optimizations
 // - Requires MTModels and MCRams mixins as prefixes to the platform config
@@ -217,3 +229,4 @@ class FireSim16LargeBoomConfig extends Config(
   new WithFireSimConfigTweaks ++
   new boom.common.WithNLargeBooms(16) ++
   new chipyard.config.AbstractConfig)
+