From 56e1aeb4007b2b2d252c2296fd2095ecf11320ec Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 7 Jul 2020 17:18:10 -0700 Subject: [PATCH] Support FireSim diplomatic multiclock --- .../chipyard/src/main/scala/ChipTop.scala | 23 +-- .../chipyard/src/main/scala/Clocks.scala | 86 +++++++--- .../src/main/scala/ConfigFragments.scala | 18 +-- .../chipyard/src/main/scala/IOBinders.scala | 51 +++--- .../chipyard/src/main/scala/Subsystem.scala | 54 ++----- .../chipyard/src/main/scala/TestHarness.scala | 21 ++- .../src/main/scala/config/RocketConfigs.scala | 8 + .../src/main/scala/BridgeBinders.scala | 2 +- .../firechip/src/main/scala/FireSim.scala | 147 ++++++++++++++---- .../src/main/scala/FireSimMulticlockPOC.scala | 107 ------------- .../src/main/scala/TargetConfigs.scala | 12 +- .../src/test/scala/ScalaTestSuite.scala | 4 +- 12 files changed, 273 insertions(+), 260 deletions(-) delete mode 100644 generators/firechip/src/main/scala/FireSimMulticlockPOC.scala diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala index 0227feb8..ff3e2aed 100644 --- a/generators/chipyard/src/main/scala/ChipTop.scala +++ b/generators/chipyard/src/main/scala/ChipTop.scala @@ -9,12 +9,11 @@ import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGr import freechips.rocketchip.config.{Parameters, Field} import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike} import freechips.rocketchip.util.{ResetCatchAndSync} -import chipyard.config.ConfigValName._ import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple} import barstools.iocell.chisel._ -case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => LazyModule(new DigitalTop()(p))) +case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => new DigitalTop()(p)) /** @@ -31,23 +30,15 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc val harnessFunctions = ArrayBuffer.empty[TestHarnessFunction] // The system module specified by BuildSystem - val lSystem = p(BuildSystem)(p).suggestName("system") + val lSystem = LazyModule(p(BuildSystem)(p)).suggestName("system") // The systemClockSinkNode provides the implicit clock and reset for the System - private val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters())) + val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters())) + val systemClockGroup = LazyModule(new ClockGroup("system_clock")) + systemClockSinkNode := systemClockGroup.node - // clockGroupNode provides a single node which aggregates all clock groups in the design - val clockGroupNode = ClockGroupIdentityNode() - - // If the specified system has diplomatic clocks, connect it to our clockGroupNode - if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) { - lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode :*= clockGroupNode } - } - // Connect the system implicit clock node to the clockGroupNode - systemClockSinkNode := ClockGroup() := clockGroupNode - - // Drive the entire diplomatic clock network using this configured Key - clockGroupNode :*=* p(ChipyardClockKey)(this) + // Generate Clocks and Reset + p(ChipyardClockKey)(this) // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children // of ChipTop (ex: ClockGroup) do not receive clock or reset. diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala index b9508d96..9c71ed96 100644 --- a/generators/chipyard/src/main/scala/Clocks.scala +++ b/generators/chipyard/src/main/scala/Clocks.scala @@ -5,19 +5,13 @@ import chisel3._ import scala.collection.mutable.{ArrayBuffer} import freechips.rocketchip.prci._ -import freechips.rocketchip.subsystem.{BaseSubsystem} +import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey} import freechips.rocketchip.config.{Parameters, Field} -import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody} -import freechips.rocketchip.util.{ResetCatchAndSync} -import chipyard.config.ConfigValName._ +import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody, LazyModule} +import freechips.rocketchip.util.{ResetCatchAndSync, Pow2ClockDivider} import barstools.iocell.chisel._ -import ChipyardClockDrivers._ - -case object ChipyardClockKey extends Field[ClockInstantiationFn](simpleTestHarnessClock) - - /** * Chipyard provides three baseline, top-level reset schemes, set using the * [[GlobalResetSchemeKey]] in a Parameters instance. These are: @@ -75,7 +69,7 @@ object GenerateReset { } reset_io.suggestName("reset") chiptop.iocells ++= resetIOCell - chiptop.harnessFunctions += ((th: TestHarness) => { + chiptop.harnessFunctions += ((th: HasHarnessUtils) => { reset_io := th.dutReset Nil }) @@ -83,33 +77,89 @@ object GenerateReset { } } -object ChipyardClockDrivers { - type ClockInstantiationFn = ChipTop => OutwardNodeHandle[ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle] +case object ChipyardClockKey extends Field[ChipTop => Unit](ClockDrivers.harnessClock) + + + +object ClockDrivers { // A simple clock provider, for testing. All clocks in system are aggregated into one, // and are driven by directly punching out to the TestHarness clock - val simpleTestHarnessClock: ClockInstantiationFn = { chiptop => + val harnessClock: ChipTop => Unit = { chiptop => implicit val p = chiptop.p val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters())) + val clockAggregator = LazyModule(new ClockGroupAggregator("clocks")) + + // Aggregate all 3 possible clock groups with the clockAggregator + chiptop.systemClockGroup.node := clockAggregator.node + if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) { + chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := clockAggregator.node } + } + chiptop.lSystem match { + case l: ChipyardSubsystem => l.tileClockGroupNode := clockAggregator.node + case _ => + } + + + clockAggregator.node := simpleClockGroupSourceNode InModuleBody { // this needs directionality so generateIOFromSignal works val clock_wire = Wire(Input(Clock())) val reset_wire = GenerateReset(chiptop, clock_wire) val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock")) chiptop.iocells ++= clockIOCell - clock_io.suggestName("clock") simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o => o.clock := clock_wire o.reset := reset_wire } - - chiptop.harnessFunctions += ((th: TestHarness) => { - clock_io := th.clock + chiptop.harnessFunctions += ((th: HasHarnessUtils) => { + clock_io := th.harnessClock Nil }) } - ClockGroupAggregator() := simpleClockGroupSourceNode + } + + val harnessMultiClock: ChipTop => Unit = { chiptop => + implicit val p = chiptop.p + val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters(), ClockGroupSourceParameters())) + val uncoreClockAggregator = LazyModule(new ClockGroupAggregator("uncore_clocks")) + + // Aggregate only the uncoreclocks + chiptop.systemClockGroup.node := uncoreClockAggregator.node + if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) { + chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := uncoreClockAggregator.node } + } + + uncoreClockAggregator.node := simpleClockGroupSourceNode + chiptop.lSystem match { + case l: ChipyardSubsystem => l.tileClockGroupNode := simpleClockGroupSourceNode + case _ => throw new Exception("MultiClock assumes ChipyardSystem") + } + + InModuleBody { + // this needs directionality so generateIOFromSignal works + val clock_wire = Wire(Input(Clock())) + val reset_wire = GenerateReset(chiptop, clock_wire) + val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock")) + chiptop.iocells ++= clockIOCell + clock_io.suggestName("clock") + val div_clock = Pow2ClockDivider(clock_wire, 2) + + simpleClockGroupSourceNode.out(0)._1.member.map { o => + o.clock := div_clock + o.reset := ResetCatchAndSync(div_clock, reset_wire.asBool) + } + simpleClockGroupSourceNode.out(1)._1.member.map { o => + o.clock := clock_wire + o.reset := reset_wire + } + chiptop.harnessFunctions += ((th: HasHarnessUtils) => { + clock_io := th.harnessClock + Nil + }) + } + } } diff --git a/generators/chipyard/src/main/scala/ConfigFragments.scala b/generators/chipyard/src/main/scala/ConfigFragments.scala index ac70f8e5..60ccc999 100644 --- a/generators/chipyard/src/main/scala/ConfigFragments.scala +++ b/generators/chipyard/src/main/scala/ConfigFragments.scala @@ -25,15 +25,8 @@ import sifive.blocks.devices.gpio._ import sifive.blocks.devices.uart._ import sifive.blocks.devices.spi._ -import chipyard.{BuildTop, BuildSystem} +import chipyard.{BuildTop, BuildSystem, ClockDrivers, ChipyardClockKey} -/** - * TODO: Why do we need this? - */ -object ConfigValName { - implicit val valName = ValName("TestHarness") -} -import ConfigValName._ // ----------------------- // Common Config Fragments @@ -73,7 +66,7 @@ class WithL2TLBs(entries: Int) extends Config((site, here, up) => { }) class WithTracegenSystem extends Config((site, here, up) => { - case BuildSystem => (p: Parameters) => LazyModule(new TraceGenSystem()(p)) + case BuildSystem => (p: Parameters) => new TraceGenSystem()(p) }) /** @@ -103,7 +96,8 @@ class WithMultiRoCCHwacha(harts: Int*) extends Config((site, here, up) => { case MultiRoCCKey => { up(MultiRoCCKey, site) ++ harts.distinct.map{ i => (i -> Seq((p: Parameters) => { - LazyModule(new Hwacha()(p)).suggestName("hwacha") + val hwacha = LazyModule(new Hwacha()(p)) + hwacha })) } } @@ -150,3 +144,7 @@ class WithRocketDCacheScratchpad extends Config((site, here, up) => { class WithNoSubsystemDrivenClocks extends Config((site, here, up) => { case SubsystemDriveAsyncClockGroupsKey => None }) + +class WithTileMultiClock extends Config((site, here, up) => { + case ChipyardClockKey => ClockDrivers.harnessMultiClock +}) diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala index 3f986757..84ef5269 100644 --- a/generators/chipyard/src/main/scala/IOBinders.scala +++ b/generators/chipyard/src/main/scala/IOBinders.scala @@ -41,7 +41,7 @@ import scala.reflect.{ClassTag} // DOC include start: IOBinders // This type describes a function callable on the TestHarness instance. Its return type is unused. -type TestHarnessFunction = (chipyard.TestHarness) => Seq[Any] +type TestHarnessFunction = (chipyard.HasHarnessUtils) => Seq[Any] // IOBinders will return a Seq of this tuple, which contains three fields: // 1. A Seq containing all IO ports created by the IOBinder function // 2. A Seq containing all IO cell modules created by the IOBinder function @@ -228,7 +228,7 @@ object AddIOCells { class WithGPIOTiedOff extends OverrideIOBinder({ (system: HasPeripheryGPIOModuleImp) => { val (ports2d, ioCells2d) = AddIOCells.gpio(system.gpio) - val harnessFn = (th: chipyard.TestHarness) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil } + val harnessFn = (th: HasHarnessUtils) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil } Seq((ports2d.flatten, ioCells2d.flatten, Some(harnessFn))) } }) @@ -237,7 +237,7 @@ class WithGPIOTiedOff extends OverrideIOBinder({ class WithUARTAdapter extends OverrideIOBinder({ (system: HasPeripheryUARTModuleImp) => { val (ports, ioCells2d) = AddIOCells.uart(system.uart) - val harnessFn = (th: chipyard.TestHarness) => { UARTAdapter.connect(ports)(system.p); Nil } + val harnessFn = (th: HasHarnessUtils) => { UARTAdapter.connect(ports)(system.p); Nil } Seq((ports, ioCells2d.flatten, Some(harnessFn))) } }) @@ -245,7 +245,7 @@ class WithUARTAdapter extends OverrideIOBinder({ class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({ (system: HasPeripherySPIFlashModuleImp) => { val (ports, ioCells2d) = AddIOCells.spi(system.qspi, "qspi") - val harnessFn = (th: chipyard.TestHarness) => { SimSPIFlashModel.connect(ports, th.reset, rdOnly)(system.p); Nil } + val harnessFn = (th: HasHarnessUtils) => { SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p); Nil } Seq((ports, ioCells2d.flatten, Some(harnessFn))) } }) @@ -253,8 +253,9 @@ class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({ class WithSimBlockDevice extends OverrideIOBinder({ (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev => val (port, ios) = AddIOCells.blockDev(bdev) - val harnessFn = (th: chipyard.TestHarness) => { - SimBlockDevice.connect(th.clock, th.reset.asBool, Some(port))(system.p) + val harnessFn = (th: HasHarnessUtils) => { + // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock + SimBlockDevice.connect(th.harnessClock, th.harnessReset.asBool, Some(port))(system.p) Nil } Seq((Seq(port), ios, Some(harnessFn))) @@ -264,7 +265,7 @@ class WithSimBlockDevice extends OverrideIOBinder({ class WithBlockDeviceModel extends OverrideIOBinder({ (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev => val (port, ios) = AddIOCells.blockDev(bdev) - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { BlockDeviceModel.connect(Some(port))(system.p) Nil } @@ -287,7 +288,7 @@ class WithSimAXIMem extends OverrideIOBinder({ val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem") // TODO: we are inlining the connectMem method of SimAXIMem because // it takes in a dut rather than seq of axi4 ports - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { peiTuples.map { case (port, edge, ios) => val mem = LazyModule(new SimAXIMem(edge, size = p(ExtMem).get.master.size)) Module(mem.module).suggestName("mem") @@ -304,14 +305,15 @@ class WithBlackBoxSimMem extends OverrideIOBinder({ (system: CanHaveMasterAXI4MemPort) => { implicit val p: Parameters = GetSystemParameters(system) val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem") - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { peiTuples.map { case (port, edge, ios) => val memSize = p(ExtMem).get.master.size val lineSize = p(CacheBlockBytes) val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle)) mem.io.axi <> port - mem.io.clock := th.clock - mem.io.reset := th.reset + // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock + mem.io.clock := th.harnessClock + mem.io.reset := th.harnessReset } Nil } @@ -323,7 +325,7 @@ class WithSimAXIMMIO extends OverrideIOBinder({ (system: CanHaveMasterAXI4MMIOPort) => { implicit val p: Parameters = GetSystemParameters(system) val peiTuples = AddIOCells.axi4(system.mmio_axi4, system.mmioAXI4Node, "mmio_mem") - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { peiTuples.zipWithIndex.map { case ((port, edge, ios), i) => val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096)) Module(mmio_mem.module).suggestName(s"mmio_mem_${i}") @@ -343,7 +345,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({ (system: HasExtInterruptsModuleImp) => { val (port, ioCells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts")) port.suggestName("interrupts") - val harnessFn = (th: chipyard.TestHarness) => { port := 0.U; Nil } + val harnessFn = (th: HasHarnessUtils) => { port := 0.U; Nil } Seq((Seq(port), ioCells, Some(harnessFn))) } }) @@ -351,7 +353,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({ class WithTieOffL2FBusAXI extends OverrideIOBinder({ (system: CanHaveSlaveAXI4Port) => { val peiTuples = AddIOCells.axi4(system.l2_frontend_bus_axi4, system.l2FrontendAXI4Node, "l2_fbus") - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { peiTuples.zipWithIndex.map { case ((port, edge, ios), i) => port := DontCare // tieoff doesn't completely tie-off, for some reason port.tieoff() @@ -366,13 +368,14 @@ class WithTiedOffDebug extends OverrideIOBinder({ (system: HasPeripheryDebugModuleImp) => { val (psdPort, resetctrlOpt, debugPortOpt, ioCells) = AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p) - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { Debug.tieoffDebug(debugPortOpt, resetctrlOpt, Some(psdPort))(system.p) // tieoffDebug doesn't actually tie everything off :/ debugPortOpt.foreach { d => - d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.clock }) + // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock + d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.harnessClock }) d.dmactiveAck := DontCare - d.clock := th.clock + d.clock := th.harnessClock } Nil } @@ -384,11 +387,11 @@ class WithSimDebug extends OverrideIOBinder({ (system: HasPeripheryDebugModuleImp) => { val (psdPort, resetctrlPortOpt, debugPortOpt, ioCells) = AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p) - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { val dtm_success = Wire(Bool()) - Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.clock, th.harnessReset, dtm_success)(system.p) + Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.harnessClock, th.harnessReset.asBool, dtm_success)(system.p) when (dtm_success) { th.success := true.B } - th.dutReset := th.harnessReset | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B) + th.dutReset := th.harnessReset.asBool | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B) Nil } Seq((Seq(psdPort) ++ debugPortOpt.toSeq, ioCells, Some(harnessFn))) @@ -398,7 +401,7 @@ class WithSimDebug extends OverrideIOBinder({ class WithTiedOffSerial extends OverrideIOBinder({ (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial => val (port, ioCells) = AddIOCells.serial(serial) - val harnessFn = (th: chipyard.TestHarness) => { + val harnessFn = (th: HasHarnessUtils) => { SerialAdapter.tieoff(port) Nil } @@ -409,8 +412,8 @@ class WithTiedOffSerial extends OverrideIOBinder({ class WithSimSerial extends OverrideIOBinder({ (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial => val (port, ioCells) = AddIOCells.serial(serial) - val harnessFn = (th: chipyard.TestHarness) => { - val ser_success = SerialAdapter.connectSimSerial(port, th.clock, th.harnessReset) + val harnessFn = (th: HasHarnessUtils) => { + val ser_success = SerialAdapter.connectSimSerial(port, th.harnessClock, th.harnessReset) when (ser_success) { th.success := true.B } Nil } @@ -422,7 +425,7 @@ class WithTraceGenSuccessBinder extends OverrideIOBinder({ (system: TraceGenSystemModuleImp) => { val (successPort, ioCells) = IOCell.generateIOFromSignal(system.success, Some("iocell_success")) successPort.suggestName("success") - val harnessFn = (th: chipyard.TestHarness) => { when (successPort) { th.success := true.B }; Nil } + val harnessFn = (th: HasHarnessUtils) => { when (successPort) { th.success := true.B }; Nil } Seq((Seq(successPort), ioCells, Some(harnessFn))) } }) diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala index 50714cc4..65ceddc9 100644 --- a/generators/chipyard/src/main/scala/Subsystem.scala +++ b/generators/chipyard/src/main/scala/Subsystem.scala @@ -11,7 +11,7 @@ import chisel3.internal.sourceinfo.{SourceInfo} import freechips.rocketchip.prci._ import freechips.rocketchip.config.{Field, Parameters} import freechips.rocketchip.devices.tilelink._ -import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp, ExportDebug} +import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp} import freechips.rocketchip.diplomacy._ import freechips.rocketchip.diplomaticobjectmodel.model.{OMInterrupt} import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{RocketTileLogicalTreeNode, LogicalModuleTree} @@ -25,59 +25,24 @@ import freechips.rocketchip.amba.axi4._ import boom.common.{BoomTile} -import testchipip.{DromajoHelper, CanHavePeripherySerial, SerialKey} - - -trait CanHaveHTIF { this: BaseSubsystem => - // Advertise HTIF if system can communicate with fesvr - if (this match { - case _: CanHavePeripherySerial if p(SerialKey) => true - case _: HasPeripheryDebug if p(ExportDebug).protocols.nonEmpty => true - case _ => false - }) { - ResourceBinding { - val htif = new Device { - def describe(resources: ResourceBindings): Description = { - val compat = resources("compat").map(_.value) - Description("htif", Map( - "compatible" -> compat)) - } - } - Resource(htif, "compat").bind(ResourceString("ucb,htif0")) - } - } -} - - -// Controls whether tiles are driven by implicit subsystem clock, or by -// diplomatic clock graph -case object UseDiplomaticTileClocks extends Field[Boolean](false) +import testchipip.{DromajoHelper} class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem with HasTiles - with CanHaveHTIF { def coreMonitorBundles = tiles.map { case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle case b: BoomTile => b.module.core.coreMonitorBundle }.toList - // TODO: In the future, RC tiles may extend ClockDomain. When that happens, - // we won't need to manually create this clock node and connect it to the - // tiles' implicit clocks - - val tilesClockSinkNode = if (p(UseDiplomaticTileClocks)) { - val node = ClockSinkNode(List(ClockSinkParameters())) - node := ClockGroup()(p, ValName("chipyard_tiles")) := asyncClockGroupsNode - Some(node) - } else { - None - } + val tileClockSinkNode = ClockSinkNode(List(ClockSinkParameters())) + val tileClockGroup = LazyModule(new ClockGroup("tile_clock")) + val tileClockGroupNode = tileClockGroup.node + tileClockSinkNode := tileClockGroupNode override lazy val module = new ChipyardSubsystemModuleImp(this) } - class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer) with HasResetVectorWire with HasTilesModuleImp @@ -87,10 +52,9 @@ class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends Bas wire.hartid := outer.hartIdList(i).U wire.reset_vector := global_reset_vector - outer.tilesClockSinkNode.map( n => { - outer.tiles(i).module.clock := n.in.head._1.clock - outer.tiles(i).module.reset := n.in.head._1.reset - }) + + outer.tiles(i).module.clock := outer.tileClockSinkNode.in.head._1.clock + outer.tiles(i).module.reset := outer.tileClockSinkNode.in.head._1.reset } // create file with core params diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala index 9344cadf..92b1dd29 100644 --- a/generators/chipyard/src/main/scala/TestHarness.scala +++ b/generators/chipyard/src/main/scala/TestHarness.scala @@ -5,34 +5,41 @@ import chisel3._ import freechips.rocketchip.diplomacy.{LazyModule} import freechips.rocketchip.config.{Field, Parameters} import chipyard.iobinders.{TestHarnessFunction} -import chipyard.config.ConfigValName._ // ------------------------------- // Chipyard Test Harness // ------------------------------- -case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => LazyModule(new ChipTop()(p))) +case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => new ChipTop()(p)) trait HasTestHarnessFunctions { val harnessFunctions: Seq[TestHarnessFunction] } -class TestHarness(implicit val p: Parameters) extends Module { +trait HasHarnessUtils { + val harnessClock: Clock + val harnessReset: Reset + val dutReset: Reset + val success: Bool +} + +class TestHarness(implicit val p: Parameters) extends Module with HasHarnessUtils { val io = IO(new Bundle { val success = Output(Bool()) }) - val ldut = p(BuildTop)(p) + val ldut = LazyModule(p(BuildTop)(p)).suggestName("ChipTop") val dut = Module(ldut.module) io.success := false.B + val harnessClock = clock + val harnessReset = WireInit(reset) + val success = io.success + // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset) ldut.harnessFunctions.foreach(_(this)) - def success = io.success - def harnessReset = this.reset.asBool - } diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala index 8e6e4867..af0b06b7 100644 --- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala +++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala @@ -183,3 +183,11 @@ class MMIORocketConfig extends Config( new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port new freechips.rocketchip.subsystem.WithNBigCores(1) ++ new chipyard.config.AbstractConfig) + +// NOTE: This config doesn't work yet because SimWidgets in the TestHarness +// always get the TestHarness clock. The Tiles and Uncore receive the correct clocks +class MultiClockRocketConfig extends Config( + new chipyard.config.WithTileMultiClock ++ // Put the Tile on its own clock domain + new freechips.rocketchip.subsystem.WithRationalRocketTiles ++ // Add rational crossings between RocketTile and uncore + new freechips.rocketchip.subsystem.WithNBigCores(1) ++ + new chipyard.config.AbstractConfig) diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala index eba57451..b59d477d 100644 --- a/generators/firechip/src/main/scala/BridgeBinders.scala +++ b/generators/firechip/src/main/scala/BridgeBinders.scala @@ -58,7 +58,7 @@ class WithBlockDeviceBridge extends OverrideIOBinder({ class WithFASEDBridge extends OverrideIOBinder({ (system: CanHaveMasterAXI4MemPort) => { implicit val p: Parameters = GetSystemParameters(system) - (system.mem_axi4 zip system.memAXI4Node.in).foreach({ case (axi4, (_, edge)) => + (system.mem_axi4 zip system.memAXI4Node.edges.in).foreach({ case (axi4, edge) => val nastiKey = NastiParameters(axi4.r.bits.data.getWidth, axi4.ar.bits.addr.getWidth, axi4.ar.bits.id.getWidth) diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala index a4cea5ec..d35dd7b6 100644 --- a/generators/firechip/src/main/scala/FireSim.scala +++ b/generators/firechip/src/main/scala/FireSim.scala @@ -3,13 +3,17 @@ package firesim.firesim import chisel3._ +import chisel3.experimental.{IO} +import freechips.rocketchip.prci._ +import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey} import freechips.rocketchip.config.{Field, Config, Parameters} -import freechips.rocketchip.diplomacy.{LazyModule} +import freechips.rocketchip.diplomacy.{LazyModule, InModuleBody} +import freechips.rocketchip.util.{ResetCatchAndSync} -import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge} +import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock} -import chipyard.{BuildSystem} +import chipyard.{BuildSystem, BuildTop, HasHarnessUtils, ChipyardSubsystem, ChipyardClockKey, ChipTop} import chipyard.iobinders.{IOBinders} // Determines the number of times to instantiate the DUT in the harness. @@ -20,6 +24,16 @@ class WithNumNodes(n: Int) extends Config((pname, site, here) => { case NumNodes => n }) +// Note, the main prerequisite for supporting an additional clock domain in a +// FireSim simulation is to supply an additional clock parameter +// (RationalClock) to the clock bridge (RationalClockBridge). The bridge +// produces a vector of clocks, based on the provided parameter list, which you +// may use freely without further modifications to your target design. +case class FireSimClockParameters(additionalClocks: Seq[RationalClock]) { + def numClocks(): Int = additionalClocks.size + 1 +} +case object FireSimClockKey extends Field[FireSimClockParameters](FireSimClockParameters(Seq())) + // Hacky: Set before each node is generated. Ideally we'd give IO binders // accesses to the the Harness's parameters instance. We could then alter that. object NodeIdx { @@ -28,33 +42,108 @@ object NodeIdx { def apply(): Int = idx } -class FireSim(implicit val p: Parameters) extends RawModule { - freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary()) - val clockBridge = Module(new RationalClockBridge) - val clock = clockBridge.io.clocks.head - val reset = WireInit(false.B) - withClockAndReset(clock, reset) { - // Instantiate multiple instances of the DUT to implement supernode - val targets = Seq.fill(p(NumNodes)) { - // It's not a RC bump without some hacks... - // Copy the AsyncClockGroupsKey to generate a fresh node on each - // instantiation of the dut, otherwise the initial instance will be - // reused across each node - import freechips.rocketchip.subsystem.AsyncClockGroupsKey - val lazyModule = p(BuildSystem)(p.alterPartial({ - case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy - })) - (lazyModule, Module(lazyModule.module)) - } +class WithFireSimSimpleClocks extends Config((site, here, up) => { + case ChipyardClockKey => { chiptop: ChipTop => + implicit val p = chiptop.p + val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters())) + val clockAggregator = LazyModule(new ClockGroupAggregator("clocks")) - val peekPokeBridge = PeekPokeBridge(clock, reset) - // A Seq of partial functions that will instantiate the right bridge only - // if that Mixin trait is present in the target's LazyModule class instance - // - // Apply each partial function to each DUT instance - for ((lazyModule, module) <- targets) { - p(IOBinders).values.foreach(f => f(lazyModule) ++ f(module)) - NodeIdx.increment() + // Aggregate all 3 possible clock groups with the clockAggregator + chiptop.systemClockGroup.node := clockAggregator.node + if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) { + chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := clockAggregator.node } + } + chiptop.lSystem match { case l: ChipyardSubsystem => l.tileClockGroupNode := clockAggregator.node } + + clockAggregator.node := simpleClockGroupSourceNode + InModuleBody { + val clock = IO(Input(Clock())).suggestName("clock") + val reset = IO(Input(Reset())).suggestName("reset") + + simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o => + o.clock := clock + o.reset := reset + } + + chiptop.harnessFunctions += ((th: HasHarnessUtils) => { + clock := th.harnessClock + reset := th.harnessReset + Nil + }) } } +}) + +class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Config((site, here, up) => { + case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor))) + case ChipyardClockKey => { chiptop: ChipTop => + implicit val p = chiptop.p + val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters(), ClockGroupSourceParameters())) + val uncoreClockAggregator = LazyModule(new ClockGroupAggregator("uncore_clocks")) + + // Aggregate only the uncoreclocks + chiptop.systemClockGroup.node := uncoreClockAggregator.node + if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) { + chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := uncoreClockAggregator.node } + } + + uncoreClockAggregator.node := simpleClockGroupSourceNode + chiptop.lSystem match { + case l: ChipyardSubsystem => l.tileClockGroupNode := simpleClockGroupSourceNode + case _ => throw new Exception("MultiClock assumes ChipyardSystem") + } + + InModuleBody { + val uncore_clock = IO(Input(Clock())).suggestName("uncore_clock") + val tile_clock = IO(Input(Clock())).suggestName("tile_clock") + val reset = IO(Input(Reset())).suggestName("reset") + + simpleClockGroupSourceNode.out(0)._1.member.map { o => + o.clock := uncore_clock + o.reset := reset + } + + simpleClockGroupSourceNode.out(1)._1.member.map { o => + o.clock := tile_clock + o.reset := ResetCatchAndSync(tile_clock, reset.asBool) + } + + chiptop.harnessFunctions += ((th: HasHarnessUtils) => { + uncore_clock := th.harnessClock + reset := th.harnessReset + th match { + case f: FireSim => tile_clock := f.additionalClocks(0) + case _ => throw new Exception("FireSimMultiClock must be used with FireSim") + } + Nil + }) + } + } +}) + +class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessUtils { + freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary()) + val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*)) + val harnessClock = clockBridge.io.clocks.head // This is the reference clock + val additionalClocks = clockBridge.io.clocks.tail + val harnessReset = WireInit(false.B) + val peekPokeBridge = PeekPokeBridge(harnessClock, harnessReset) + val dutReset = false.B // unused (if used, its a bug) + val success = false.B // unused (if used, its a bug) + + // Instantiate multiple instances of the DUT to implement supernode + for (i <- 0 until p(NumNodes)) { + // It's not a RC bump without some hacks... + // Copy the AsyncClockGroupsKey to generate a fresh node on each + // instantiation of the dut, otherwise the initial instance will be + // reused across each node + import freechips.rocketchip.subsystem.AsyncClockGroupsKey + val lazyModule = LazyModule(p(BuildTop)(p.alterPartial({ + case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy + }))) + val module = Module(lazyModule.module) + require(lazyModule.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset") + lazyModule.harnessFunctions.foreach(_(this)) + NodeIdx.increment() + } } diff --git a/generators/firechip/src/main/scala/FireSimMulticlockPOC.scala b/generators/firechip/src/main/scala/FireSimMulticlockPOC.scala deleted file mode 100644 index bf0e0e26..00000000 --- a/generators/firechip/src/main/scala/FireSimMulticlockPOC.scala +++ /dev/null @@ -1,107 +0,0 @@ -//See LICENSE for license details. - -package firesim.firesim - -import chisel3._ - -import freechips.rocketchip.config.{Field, Config, Parameters} -import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, RationalCrossing} -import freechips.rocketchip.subsystem._ -import freechips.rocketchip.util.{ResetCatchAndSync} - -import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock} -import firesim.configs._ - -import boom.common.{WithRationalBoomTiles} - -import chipyard.{BuildSystem, DigitalTop, DigitalTopModule} -import chipyard.config.ConfigValName._ -import chipyard.iobinders.{IOBinders} - -// WIP! This file is a sketch of one means of defining a multiclock target-design -// that can be simulated in FireSim, pending a canonicalized form in Chipyard. -// -// Note, the main prerequisite for supporting an additional clock domain in a -// FireSim simulation is to supply an additional clock parameter -// (RationalClock) to the clock bridge (RationalClockBridge). The bridge -// produces a vector of clocks, based on the provided parameter list, which you -// may use freely without further modifications to your target design. - -case class FireSimClockParameters(additionalClocks: Seq[RationalClock]) { - def numClocks(): Int = additionalClocks.size + 1 -} -case object FireSimClockKey extends Field[FireSimClockParameters](FireSimClockParameters(Seq())) - -trait HasAdditionalClocks extends LazyModuleImp { - val clocks = IO(Vec(p(FireSimClockKey).numClocks, Input(Clock()))) -} - -// Presupposes only 1 or 2 clocks. -trait HasFireSimClockingImp extends HasAdditionalClocks { - val outer: HasTiles - val (tileClock, tileReset) = p(FireSimClockKey).additionalClocks.headOption match { - case Some(RationalClock(_, numer, denom)) if numer != denom => (clocks(1), ResetCatchAndSync(clocks(1), reset.toBool)) - case None => (clocks.head, reset) - } - - outer.tiles.foreach({ case tile => - tile.module.clock := tileClock - tile.module.reset := tileReset - }) -} - -// Config Fragment -class WithSingleRationalTileDomain(multiplier: Int, divisor: Int) extends Config( - new WithRationalRocketTiles ++ - new WithRationalBoomTiles ++ - new Config((site, here, up) => { - case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor))) - }) -) - -class HalfRateUncore extends WithSingleRationalTileDomain(2,1) - -class WithFiresimMulticlockTop extends Config((site, here, up) => { - case BuildSystem => (p: Parameters) => LazyModule(new FiresimMulticlockTop()(p)).suggestName("system") -}) - -// Complete Config -class FireSimQuadRocketMulticlockConfig extends Config( - new HalfRateUncore ++ - new WithFiresimMulticlockTop ++ - new FireSimQuadRocketConfig) - -// Top Definition -class FiresimMulticlockTop(implicit p: Parameters) extends chipyard.DigitalTop -{ - override lazy val module = new FiresimMulticlockTopModule(this) -} - - -class FiresimMulticlockTopModule[+L <: DigitalTop](l: L) extends chipyard.DigitalTopModule(l) with HasFireSimClockingImp - -// Harness Definition -class FireSimMulticlockPOC(implicit val p: Parameters) extends RawModule { - freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary()) - val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*)) - val refClock = clockBridge.io.clocks.head - val reset = WireInit(false.B) - withClockAndReset(refClock, reset) { - // Instantiate multiple instances of the DUT to implement supernode - val targets = Seq.fill(p(NumNodes)) { - val lazyModule = p(BuildSystem)(p) - (lazyModule, Module(lazyModule.module)) - } - val peekPokeBridge = PeekPokeBridge(refClock, reset) - // A Seq of partial functions that will instantiate the right bridge only - // if that Mixin trait is present in the target's class instance - // - // Apply each partial function to each DUT instance - for ((lazyModule, module) <- targets) { - p(IOBinders).values.foreach(f => f(lazyModule) ++ f(module)) - } - targets.collect({ case (_, t: HasAdditionalClocks) => t.clocks := clockBridge.io.clocks }) - } -} - - diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala index 2828580a..d9ba92ec 100644 --- a/generators/firechip/src/main/scala/TargetConfigs.scala +++ b/generators/firechip/src/main/scala/TargetConfigs.scala @@ -22,7 +22,6 @@ import testchipip.WithRingSystemBus import firesim.bridges._ import firesim.configs._ -import chipyard.config.ConfigValName._ class WithBootROM extends Config((site, here, up) => { case BootROMParams => { @@ -67,6 +66,8 @@ class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small") // Tweaks that are generally applied to all firesim configs class WithFireSimConfigTweaks extends Config( + // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset + new WithFireSimSimpleClocks ++ // Required*: When using FireSim-as-top to provide a correct path to the target bootrom source new WithBootROM ++ // Optional*: Removing this will require target-software changes to properly capture UART output @@ -170,3 +171,12 @@ class FireSimArianeConfig extends Config( new WithDefaultMemModel ++ new WithFireSimConfigTweaks ++ new chipyard.ArianeConfig) + + +class FireSimMulticlockRocketConfig extends Config( + new WithFireSimRationalTileDomain(2, 1) ++ + new WithDefaultFireSimBridges ++ + new WithDefaultMemModel ++ + new WithFireSimConfigTweaks ++ + new chipyard.MultiClockRocketConfig) + diff --git a/generators/firechip/src/test/scala/ScalaTestSuite.scala b/generators/firechip/src/test/scala/ScalaTestSuite.scala index f92a7960..ea1627b7 100644 --- a/generators/firechip/src/test/scala/ScalaTestSuite.scala +++ b/generators/firechip/src/test/scala/ScalaTestSuite.scala @@ -106,8 +106,8 @@ class BoomF1Tests extends FireSimTestSuite("FireSim", "DDR3FRFCFSLLC4MB_FireSimL class RocketNICF1Tests extends FireSimTestSuite("FireSim", "WithNIC_DDR3FRFCFSLLC4MB_FireSimRocketConfig", "BaseF1Config") // Multiclock tests class RocketMulticlockF1Tests extends FireSimTestSuite( - "FireSimMulticlockPOC", - "FireSimQuadRocketMulticlockConfig", + "FireSim", + "FireSimMulticlockRocketConfig", "WithSynthAsserts_BaseF1Config") class ArianeF1Tests extends FireSimTestSuite("FireSim", "WithNIC_DDR3FRFCFSLLC4MB_FireSimArianeConfig", "BaseF1Config")