wu-arch/chipyard
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Unify multi-node btw chipyard/firechip | unify harness clocking

Browse Source
This commit is contained in:
Jerry Zhao
2023-05-12 08:21:18 -07:00
parent 0cbca54e19
commit 607c2b5a73
31 changed files with 441 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
generators/firechip/src/main/scala/BridgeBinders.scala
View File

@@ -4,6 +4,7 @@ package firesim.firesim
import chisel3._
import chisel3.experimental.annotate
import chisel3.experimental.{DataMirror, Direction}
import chisel3.util.experimental.BoringUtils
import org.chipsalliance.cde.config.{Field, Config, Parameters}
@@ -30,11 +31,12 @@ import cva6.CVA6Tile
import boom.common.{BoomTile}
import barstools.iocell.chisel._
import chipyard.iobinders.{IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters, IOCellKey}
import chipyard._
import chipyard.harness._
object MainMemoryConsts {
val regionNamePrefix = "MainMemory"
def globalName = s"${regionNamePrefix}_${NodeIdx()}"
def globalName()(implicit p: Parameters) = s"${regionNamePrefix}_${p(MultiChipIdx)}"
}
trait Unsupported {
@@ -72,11 +74,9 @@ class WithTSIBridgeAndHarnessRAMOverSerialTL extends OverrideHarnessBinder({
ports.map { port =>
implicit val p = GetSystemParameters(system)
val bits = port.bits
port.clock := th.buildtopClock
val ram = withClockAndReset(th.buildtopClock, th.buildtopReset) {
TSIHarness.connectRAM(system.serdesser.get, bits, th.buildtopReset)
}
TSIBridge(th.buildtopClock, ram.module.io.tsi, p(ExtMem).map(_ => MainMemoryConsts.globalName), th.buildtopReset.asBool)
port.clock := th.harnessBinderClock
val ram = TSIHarness.connectRAM(system.serdesser.get, bits, th.harnessBinderReset)
TSIBridge(th.harnessBinderClock, ram.module.io.tsi, p(ExtMem).map(_ => MainMemoryConsts.globalName), th.harnessBinderReset.asBool)
}
Nil
}
@@ -97,13 +97,13 @@ class WithUARTBridge extends OverrideHarnessBinder({
val pbusClockNode = system.outer.asInstanceOf[HasTileLinkLocations].locateTLBusWrapper(PBUS).fixedClockNode
val pbusClock = pbusClockNode.in.head._1.clock
BoringUtils.bore(pbusClock, Seq(uartSyncClock))
ports.map { p => UARTBridge(uartSyncClock, p, th.buildtopReset.asBool)(system.p) }; Nil
ports.map { p => UARTBridge(uartSyncClock, p, th.harnessBinderReset.asBool)(system.p) }; Nil
})
class WithBlockDeviceBridge extends OverrideHarnessBinder({
(system: CanHavePeripheryBlockDevice, th: FireSim, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
implicit val p: Parameters = GetSystemParameters(system)
ports.map { b => BlockDevBridge(b.clock, b.bits, th.buildtopReset.asBool) }
ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessBinderReset.asBool) }
Nil
}
})
@@ -120,21 +120,16 @@ class WithAXIOverSerialTLCombinedBridges extends OverrideHarnessBinder({
val memFreq = axiDomainParams.getMemFrequency(system.asInstanceOf[HasTileLinkLocations])
ports.map({ port =>
val axiClock = p(ClockBridgeInstantiatorKey).requestClock("mem_over_serial_tl_clock", memFreq)
val axiClockBundle = Wire(new ClockBundle(ClockBundleParameters()))
axiClockBundle.clock := axiClock
axiClockBundle.reset := ResetCatchAndSync(axiClock, th.buildtopReset.asBool)
val axiClock = th.harnessClockInstantiator.requestClockHz("mem_over_serial_tl_clock", memFreq)
val serial_bits = port.bits
port.clock := th.buildtopClock
val harnessMultiClockAXIRAM = withClockAndReset(th.buildtopClock, th.buildtopReset) {
TSIHarness.connectMultiClockAXIRAM(
system.serdesser.get,
serial_bits,
axiClockBundle,
th.buildtopReset)
}
TSIBridge(th.buildtopClock, harnessMultiClockAXIRAM.module.io.tsi, Some(MainMemoryConsts.globalName), th.buildtopReset.asBool)
port.clock := th.harnessBinderClock
val harnessMultiClockAXIRAM = TSIHarness.connectMultiClockAXIRAM(
system.serdesser.get,
serial_bits,
axiClock,
ResetCatchAndSync(axiClock, th.harnessBinderReset.asBool))
TSIBridge(th.harnessBinderClock, harnessMultiClockAXIRAM.module.io.tsi, Some(MainMemoryConsts.globalName), th.harnessBinderReset.asBool)
// connect SimAxiMem
(harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi4, edge) =>
@@ -192,7 +187,7 @@ class WithDromajoBridge extends ComposeHarnessBinder({
class WithTraceGenBridge extends OverrideHarnessBinder({
(system: TraceGenSystemModuleImp, th: FireSim, ports: Seq[Bool]) =>
ports.map { p => GroundTestBridge(th.buildtopClock, p)(system.p) }; Nil
ports.map { p => GroundTestBridge(th.harnessBinderClock, p)(system.p) }; Nil
})
class WithFireSimMultiCycleRegfile extends ComposeIOBinder({

261
generators/firechip/src/main/scala/FireSim.scala
View File

@@ -20,258 +20,67 @@ import chipyard.harness._
import chipyard.iobinders._
import chipyard.clocking._
// Determines the number of times to instantiate the DUT in the harness.
// Subsumes legacy supernode support
case object NumNodes extends Field[Int](1)
class WithNumNodes(n: Int) extends Config((pname, site, here) => {
case NumNodes => n
})
// Hacky: Set before each node is generated. Ideally we'd give IO binders
// accesses to the the Harness's parameters instance. We could then alter that.
object NodeIdx {
private var idx = 0
def increment(): Unit = {idx = idx + 1 }
def apply(): Int = idx
}
/**
* Specifies DUT clocks for the rational clock bridge
*
* @param allClocks Seq. of RationalClocks that want a clock
*
* @param baseClockName Name of domain that the allClocks is rational to
*
* @param baseFreqRequested Freq. for the reference domain in Hz
*/
case class BuildTopClockParameters(allClocks: Seq[RationalClock], baseClockName: String, baseFreqRequested: Double)
/**
* Under FireSim's current multiclock implementation there can be only a
* single clock bridge. This requires, therefore, that it be instantiated in
* the harness and reused across all supernode instances. This class attempts to
* memoize its instantiation such that it can be referenced from within a ClockScheme function.
*/
class ClockBridgeInstantiator {
private val _harnessClockMap: LinkedHashMap[String, (Double, Clock)] = LinkedHashMap.empty
class FireSimClockBridgeInstantiator extends HarnessClockInstantiator {
// connect all clock wires specified to the RationalClockBridge
def instantiateHarnessClocks(refClock: Clock): Unit = {
val sinks = clockMap.map({ case (name, (freq, bundle)) =>
ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq / (1000 * 1000))), name=Some(name))
}).toSeq
// Assumes that the supernode implementation results in duplicated clocks
// (i.e. only 1 set of clocks is generated for all BuildTop designs)
private var _buildTopClockParams: Option[BuildTopClockParameters] = None
private val _buildTopClockMap: LinkedHashMap[String, (RationalClock, Clock)] = LinkedHashMap.empty
private var _buildTopClockRecord: Option[RecordMap[Clock]] = None
val pllConfig = new SimplePllConfiguration("firesimRationalClockBridge", sinks)
pllConfig.emitSummaries()
/**
* Request a clock at a particular frequency
*
* @param name An identifier for the associated clock domain
*
* @param freqRequested Freq. for the domain in Hz
*/
def requestClock(name: String, freqRequested: Double): Clock = {
val clkWire = Wire(new Clock)
_harnessClockMap(name) = (freqRequested, clkWire)
clkWire
}
/**
* Get a RecordMap of clocks for a set of input RationalClocks. Used to drive
* the design elaborated by buildtop
*
* @param clockMapParameters Defines the set of required clocks
*/
def requestClockRecordMap(clockMapParameters: BuildTopClockParameters): RecordMap[Clock] = {
if (_buildTopClockParams.isDefined) {
require(_buildTopClockParams.get == clockMapParameters, "Must request same set of clocks on repeated invocations.")
} else {
val clockRecord = Wire(RecordMap(clockMapParameters.allClocks.map { c => (c.name, Clock()) }:_*))
// Build up the mutable structures describing the clocks for the dut
_buildTopClockParams = Some(clockMapParameters)
_buildTopClockRecord = Some(clockRecord)
for (clock <- clockMapParameters.allClocks) {
val clockWire = Wire(new Clock)
_buildTopClockMap(clock.name) = (clock, clockWire)
clockRecord(clock.name).get := clockWire
var instantiatedClocks = LinkedHashMap[Int, (Clock, Seq[String])]()
// connect wires to clock source
for ((name, (freq, clock)) <- clockMap) {
val freqMHz = (freq / (1000 * 1000)).toInt
if (!instantiatedClocks.contains(freqMHz)) {
val clock = Wire(Clock())
instantiatedClocks(freqMHz) = (clock, Seq(name))
} else {
instantiatedClocks(freqMHz) = (instantiatedClocks(freqMHz)._1, instantiatedClocks(freqMHz)._2 :+ name)
}
clock := instantiatedClocks(freqMHz)._1
}
_buildTopClockRecord.get
}
/**
* Connect all clocks requested to ClockBridge
*/
def instantiateFireSimClockBridge: Unit = {
require(_buildTopClockParams.isDefined, "Must have rational clocks to assign to")
val BuildTopClockParameters(allClocks, refRatClockName, refRatClockFreq) = _buildTopClockParams.get
require(_buildTopClockMap.exists(_._1 == refRatClockName),
s"Provided base-clock name for rational clocks, ${refRatClockName}, doesn't match a name within specified rational clocks." +
"Available clocks:\n " + _buildTopClockMap.map(_._1).mkString("\n "))
// Simplify the RationalClocks ratio's
val refRatClock = _buildTopClockMap.find(_._1 == refRatClockName).get._2._1
val simpleRatClocks = _buildTopClockMap.map { t =>
val ratClock = t._2._1
ratClock.copy(
multiplier = ratClock.multiplier * refRatClock.divisor,
divisor = ratClock.divisor * refRatClock.multiplier).simplify
}
// Determine all the clock dividers (harness + rational clocks)
// Note: Requires that the BuildTop reference frequency is requested with proper freq.
val refRatSinkParams = ClockSinkParameters(take=Some(ClockParameters(freqMHz=refRatClockFreq / (1000 * 1000))),name=Some(refRatClockName))
val harSinkParams = _harnessClockMap.map { case (name, (freq, bundle)) =>
ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq / (1000 * 1000))),name=Some(name))
val ratClocks = instantiatedClocks.map { case (freqMHz, (clock, names)) =>
(RationalClock(names.mkString(","), 1, pllConfig.referenceFreqMHz.toInt / freqMHz), clock)
}.toSeq
val allSinkParams = harSinkParams :+ refRatSinkParams
// Use PLL config to determine overall div's
val pllConfig = new SimplePllConfiguration("firesimOverallClockBridge", allSinkParams)
pllConfig.emitSummaries
// Adjust all BuildTop RationalClocks with the div determined by the PLL
val refRatDiv = pllConfig.sinkDividerMap(refRatSinkParams)
val adjRefRatClocks = simpleRatClocks.map { clock =>
clock.copy(divisor = clock.divisor * refRatDiv).simplify
}
// Convert harness clocks to RationalClocks
val harRatClocks = harSinkParams.map { case ClockSinkParameters(_, _, _, _, clkParamsOpt, nameOpt) =>
RationalClock(nameOpt.get, 1, pllConfig.referenceFreqMHz.toInt / clkParamsOpt.get.freqMHz.toInt)
}
val allAdjRatClks = adjRefRatClocks ++ harRatClocks
// Removes clocks that have the same frequency before instantiating the
// clock bridge to avoid unnecessary BUFGCE use.
val allDistinctRatClocks = allAdjRatClks.foldLeft(Seq(RationalClock(pllConfig.referenceSinkParams.name.get, 1, 1))) {
case (list, candidate) => if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate
}
val clockBridge = Module(new RationalClockBridge(allDistinctRatClocks))
val cbVecTuples = allDistinctRatClocks.zip(clockBridge.io.clocks)
// Connect all clocks (harness + BuildTop clocks)
for (clock <- allAdjRatClks) {
val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(clock)).get
_buildTopClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField }
_harnessClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField }
val clockBridge = Module(new RationalClockBridge(ratClocks.map(_._1)))
(clockBridge.io.clocks zip ratClocks).foreach { case (clk, rat) =>
rat._2 := clk
}
}
}
case object ClockBridgeInstantiatorKey extends Field[ClockBridgeInstantiator](new ClockBridgeInstantiator)
case object FireSimBaseClockNameKey extends Field[String]("implicit_clock")
class ClocksWithSinkParams(val params: Seq[ClockSinkParameters]) extends Bundle {
val clocks = Vec(params.size, Clock())
}
class WithFireSimSimpleClocks extends OverrideLazyIOBinder({
(system: HasChipyardPRCI) => {
implicit val p = GetSystemParameters(system)
// Figure out what provides this in the chipyard scheme
implicit val valName = ValName("FireSimClocking")
val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
system.connectImplicitClockSinkNode(implicitClockSinkNode)
InModuleBody {
val implicit_clock = implicitClockSinkNode.in.head._1.clock
val implicit_reset = implicitClockSinkNode.in.head._1.reset
system.asInstanceOf[BaseSubsystem].module match { case l: LazyModuleImp => {
l.clock := implicit_clock
l.reset := implicit_reset
}}
}
val inputClockSource = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
system.allClockGroupsNode := inputClockSource
InModuleBody {
val (clockGroupBundle, clockGroupEdge) = inputClockSource.out.head
val reset_io = IO(Input(AsyncReset())).suggestName("async_reset")
val input_clocks = IO(Input(new ClocksWithSinkParams(clockGroupEdge.sink.members)))
.suggestName("clocks")
(clockGroupBundle.member.data zip input_clocks.clocks).foreach { case (clockBundle, inputClock) =>
clockBundle.clock := inputClock
clockBundle.reset := reset_io
}
(Seq(reset_io, input_clocks), Nil)
}
}
})
class WithFireSimHarnessClockBinder extends OverrideHarnessBinder({
(system: HasChipyardPRCI, th: FireSim, ports: Seq[Data]) => {
implicit val p = th.p
ports.map ({
case c: ClocksWithSinkParams => {
val pllConfig = new SimplePllConfiguration("firesimBuildTopClockGenerator", c.params)
pllConfig.emitSummaries
th.setRefClockFreq(pllConfig.referenceFreqMHz)
val rationalClockSpecs = for ((sinkP, division) <- pllConfig.sinkDividerMap) yield {
RationalClock(sinkP.name.get, 1, division)
}
val input_clocks: RecordMap[Clock] = p(ClockBridgeInstantiatorKey).requestClockRecordMap(
BuildTopClockParameters(
rationalClockSpecs.toSeq,
p(FireSimBaseClockNameKey),
pllConfig.referenceFreqMHz * (1000 * 1000)))
(c.clocks zip c.params) map ({ case (clock, param) =>
clock := input_clocks(param.name.get).get
})
}
case r: Reset => r := th.buildtopReset.asAsyncReset
})
}
})
class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSignalReferences {
class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessInstantiators {
require(harnessClockInstantiator.isInstanceOf[FireSimClockBridgeInstantiator])
freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
val buildtopClock = Wire(Clock())
val buildtopReset = WireInit(false.B)
// The peek-poke bridge must still be instantiated even though it's
// functionally unused. This will be removed in a future PR.
val dummy = WireInit(false.B)
val peekPokeBridge = PeekPokeBridge(buildtopClock, dummy)
val peekPokeBridge = PeekPokeBridge(harnessBinderClock, dummy)
val resetBridge = Module(new ResetPulseBridge(ResetPulseBridgeParameters()))
// In effect, the bridge counts the length of the reset in terms of this clock.
resetBridge.io.clock := buildtopClock
buildtopReset := resetBridge.io.reset
// Ensures FireSim-synthesized assertions and instrumentation is disabled
// while buildtopReset is asserted. This ensures assertions do not fire at
// time zero in the event their local reset is delayed (typically because it
// has been pipelined)
midas.targetutils.GlobalResetCondition(buildtopReset)
resetBridge.io.clock := harnessBinderClock
def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B }
def implicitClock = false.B.asClock // unused
def implicitReset = resetBridge.io.reset
def success = { require(false, "success should not be used in Firesim"); false.B }
// Instantiate multiple instances of the DUT to implement supernode
for (i <- 0 until p(NumNodes)) {
// It's not a RC bump without some hacks...
// Copy the AsyncClockGroupsKey to generate a fresh node on each
// instantiation of the dut, otherwise the initial instance will be
// reused across each node
import freechips.rocketchip.subsystem.AsyncClockGroupsKey
val lazyModule = LazyModule(p(BuildTop)(p))
val module = Module(lazyModule.module)
instantiateChipTops()
lazyModule match { case d: HasIOBinders =>
ApplyHarnessBinders(this, d.lazySystem, d.portMap)
}
NodeIdx.increment()
}
buildtopClock := p(ClockBridgeInstantiatorKey).requestClock("buildtop_reference_clock", getRefClockFreq * (1000 * 1000))
p(ClockBridgeInstantiatorKey).instantiateFireSimClockBridge
// Ensures FireSim-synthesized assertions and instrumentation is disabled
// while resetBridge.io.reset is asserted. This ensures assertions do not fire at
// time zero in the event their local reset is delayed (typically because it
// has been pipelined)
midas.targetutils.GlobalResetCondition(resetBridge.io.reset)
}

28
generators/firechip/src/main/scala/TargetConfigs.scala
View File

@@ -18,6 +18,7 @@ import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTParams}
import scala.math.{min, max}
import chipyard.clocking.{ChipyardPRCIControlKey}
import chipyard.harness.{HarnessClockInstantiatorKey}
import icenet._
import firesim.bridges._
@@ -43,6 +44,11 @@ class WithoutClockGating extends Config((site, here, up) => {
case ChipyardPRCIControlKey => up(ChipyardPRCIControlKey, site).copy(enableTileClockGating = false)
})
// Use the firesim clock bridge instantiator. this is required
class WithFireSimHarnessClockBridgeInstantiator extends Config((site, here, up) => {
case HarnessClockInstantiatorKey => () => new FireSimClockBridgeInstantiator
})
// Testing configurations
// This enables printfs used in testing
class WithScalaTestFeatures extends Config((site, here, up) => {
@@ -63,9 +69,10 @@ class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small")
// Minimal set of FireSim-related design tweaks - notably discludes FASED, TraceIO, and the BlockDevice
class WithMinimalFireSimDesignTweaks extends Config(
// Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
new WithFireSimHarnessClockBinder ++
new WithFireSimSimpleClocks ++
// Required*: Punch all clocks to FireSim's harness clock instantiator
new WithFireSimHarnessClockBridgeInstantiator ++
new chipyard.harness.WithClockAndResetFromHarness ++
new chipyard.clocking.WithPassthroughClockGenerator ++
// Required*: When using FireSim-as-top to provide a correct path to the target bootrom source
new WithBootROM ++
// Required: Existing FAME-1 transform cannot handle black-box clock gates
@@ -116,19 +123,11 @@ class WithFireSimConfigTweaks extends Config(
// Using some other frequency will require runnings the FASED runtime configuration generator
// to generate faithful DDR3 timing values.
new chipyard.config.WithSystemBusFrequency(1000.0) ++
new chipyard.config.WithSystemBusFrequencyAsDefault ++ // All unspecified clock frequencies, notably the implicit clock, will use the sbus freq (1000 MHz)
// Explicitly set PBUS + MBUS to 1000 MHz, since they will be driven to 100 MHz by default because of assignments in the Chisel
new chipyard.config.WithPeripheryBusFrequency(1000.0) ++
new chipyard.config.WithMemoryBusFrequency(1000.0) ++
new WithFireSimDesignTweaks
)
// Tweak more representative of testchip configs
class WithFireSimTestChipConfigTweaks extends Config(
new chipyard.config.WithTestChipBusFreqs ++
new WithFireSimDesignTweaks
)
// Tweaks to use minimal design tweaks
// Need to use initramfs to use linux (no block device)
class WithMinimalFireSimHighPerfConfigTweaks extends Config(
@@ -267,9 +266,10 @@ class FireSimLeanGemminiPrintfRocketConfig extends Config(
// Supernode Configurations, base off chipyard's RocketConfig
//**********************************************************************************
class SupernodeFireSimRocketConfig extends Config(
new WithNumNodes(4) ++
new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 8L) ++ // 8 GB
new FireSimRocketConfig)
new WithFireSimHarnessClockBridgeInstantiator ++
new chipyard.harness.WithHomogeneousMultiChip(n=4, new Config(
new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 8L) ++ // 8GB DRAM per node
new FireSimRocketConfig)))
//**********************************************************************************
//* CVA6 Configurations