diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala index c02df03e..4c5dd4f6 100644 --- a/generators/chipyard/src/main/scala/TestHarness.scala +++ b/generators/chipyard/src/main/scala/TestHarness.scala @@ -95,7 +95,7 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign case d: HasReferenceClockFreq => d.refClockFreqMHz.getOrElse(p(DefaultClockFrequencyKey)) case _ => p(DefaultClockFrequencyKey) } - val refClkBundle = p(HarnessClockInstantiatorKey).getClockBundleWire("chiptop_reference_clock", freqMHz * (1000 * 1000)) + val refClkBundle = p(HarnessClockInstantiatorKey).getClockBundleWire("buildtop_reference_clock", freqMHz * (1000 * 1000)) harnessClock := refClkBundle.clock harnessReset := WireInit(refClkBundle.reset) diff --git a/generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala b/generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala index 00cb6814..589d99c6 100644 --- a/generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala +++ b/generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala @@ -89,6 +89,7 @@ class SimplePllConfiguration( ElaborationArtefacts.add(s"${name}.freq-summary", summaryString) println(summaryString) } + def referenceSinkParams(): ClockSinkParameters = sinkDividerMap.find(_._2 == 1).get._1 } case class DividerOnlyClockGeneratorNode(pllName: String)(implicit valName: ValName) diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala index b6b3caf9..174ea5bb 100644 --- a/generators/firechip/src/main/scala/BridgeBinders.scala +++ b/generators/firechip/src/main/scala/BridgeBinders.scala @@ -12,6 +12,8 @@ import freechips.rocketchip.devices.debug.{Debug, HasPeripheryDebugModuleImp} import freechips.rocketchip.amba.axi4.{AXI4Bundle} import freechips.rocketchip.subsystem._ import freechips.rocketchip.tile.{RocketTile} +import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters} +import freechips.rocketchip.util.{ResetCatchAndSync} import sifive.blocks.devices.uart._ import testchipip._ @@ -104,24 +106,43 @@ class WithBlockDeviceBridge extends OverrideHarnessBinder({ }) class WithAXIOverSerialTLCombinedBridges extends OverrideHarnessBinder({ - (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[SerialAndPassthroughClockResetIO]) => { + (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[ClockedIO[SerialIO]]) => { implicit val p = GetSystemParameters(system) p(SerialTLKey).map({ sVal => - // require having memory over the serdes link + // currently only the harness AXI port supports a passthrough clock + require(sVal.axiMemOverSerialTLParams.isDefined) + val axiDomainParams = sVal.axiMemOverSerialTLParams.get require(sVal.isMemoryDevice) + val memFreq: Double = axiDomainParams.axiClockParams match { + case Some(clkParams) => clkParams.clockFreqMHz * 1000000 + case None => { + // get freq. from what the master of the serial link specifies + system.asInstanceOf[HasTileLinkLocations].locateTLBusWrapper(p(SerialTLAttachKey).masterWhere).dtsFrequency.get.toDouble + } + } + ports.map({ port => - val offchipNetwork = SerialAdapter.connectHarnessMultiClockAXIRAM(system.serdesser.get, port, th.harnessReset) - SerialBridge(port.clocked_serial.clock, offchipNetwork.module.io.tsi_ser, Some(MainMemoryConsts.globalName)) + val axiClock = p(ClockBridgeInstantiatorKey).getClock("mem_over_serial_tl_clock", memFreq) + val axiClockBundle = Wire(new ClockBundle(ClockBundleParameters())) + axiClockBundle.clock := axiClock + axiClockBundle.reset := ResetCatchAndSync(axiClock, th.harnessReset.asBool) + + val harnessMultiClockAXIRAM = SerialAdapter.connectHarnessMultiClockAXIRAM( + system.serdesser.get, + port, + axiClockBundle, + th.harnessReset) + SerialBridge(port.clock, harnessMultiClockAXIRAM.module.io.tsi_ser, Some(MainMemoryConsts.globalName)) // connect SimAxiMem - (offchipNetwork.mem_axi4 zip offchipNetwork.memAXI4Node.edges.in).map { case (axi4, edge) => - val nastiKey = NastiParameters(axi4.r.bits.data.getWidth, - axi4.ar.bits.addr.getWidth, - axi4.ar.bits.id.getWidth) + (harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi4, edge) => + val nastiKey = NastiParameters(axi4.bits.r.bits.data.getWidth, + axi4.bits.ar.bits.addr.getWidth, + axi4.bits.ar.bits.id.getWidth) system match { - case s: BaseSubsystem => FASEDBridge(port.passthrough_clock_reset.clock, axi4, port.passthrough_clock_reset.reset.asBool, + case s: BaseSubsystem => FASEDBridge(axi4.clock, axi4.bits, axi4.reset.asBool, CompleteConfig(p(firesim.configs.MemModelKey), nastiKey, Some(AXI4EdgeSummary(edge)), diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala index ff765970..b1498a5b 100644 --- a/generators/firechip/src/main/scala/FireSim.scala +++ b/generators/firechip/src/main/scala/FireSim.scala @@ -2,6 +2,8 @@ package firesim.firesim +import scala.collection.mutable.{HashMap} + import chisel3._ import chisel3.experimental.{IO} @@ -38,44 +40,104 @@ object NodeIdx { /** * Under FireSim's current multiclock implementation there can be only a * single clock bridge. This requires, therefore, that it be instantiated in - * the harness and reused across all supernode instances. This class attempts to + * the harness and reused across all supernode instances. This class attempts to * memoize its instantiation such that it can be referenced from within a ClockScheme function. */ class ClockBridgeInstantiator { - private var _clockRecord: Option[RecordMap[Clock]] = None + private var _harnessClockMap: HashMap[String, (Double, Clock)] = HashMap.empty - def getClockRecord: RecordMap[Clock] = _clockRecord.get + // Assumes that the supernode implementation results in duplicated clocks + // (i.e. only 1 set of clocks is generated for all BuildTop designs) + private var _ratClockMap: HashMap[String, (RationalClock, Clock)] = HashMap.empty + private var _ratRefName: Option[String] = None - def getClockRecordOrInstantiate(allClocks: Seq[RationalClock], baseClockName: String): RecordMap[Clock] = { - if (_clockRecord.isEmpty) { - require(allClocks.exists(_.name == baseClockName), - s"Provided base-clock name, ${baseClockName}, does not match a defined clock. Available clocks:\n " + - allClocks.map(_.name).mkString("\n ")) + /** + * Request a clock at a particular frequency + * + * @param name An identifier for the associated clock domain + * + * @param freqRequested Freq. for the domain in Hz + */ + def getClock(name: String, freqRequested: Double): Clock = { + val clkWire = Wire(new Clock) + _harnessClockMap(name) = (freqRequested, clkWire) + clkWire + } - val baseClock = allClocks.find(_.name == baseClockName).get - val simplified = allClocks.map { c => - c.copy(multiplier = c.multiplier * baseClock.divisor, divisor = c.divisor * baseClock.multiplier) - .simplify - } + /** + * Get a RecordMap of clocks for a set of input RationalClocks + * + * @param allClocks Seq. of RationalClocks that want a clock + * + * @param baseClockName Name of domain that the allClocks is rational to + */ + def getClockRecordMap(allClocks: Seq[RationalClock], baseClockName: String): RecordMap[Clock] = { + val ratClockRecordMapWire = Wire(RecordMap(allClocks.map { c => (c.name, Clock()) }:_*)) - /** - * Removes clocks that have the same frequency before instantiating the - * clock bridge to avoid unnecessary BUFGCE use. - */ - val distinct = simplified.foldLeft(Seq(RationalClock(baseClockName, 1, 1))) { case (list, candidate) => - if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate - } - - val clockBridge = Module(new RationalClockBridge(distinct)) - val cbVecTuples = distinct.zip(clockBridge.io.clocks) - val outputWire = Wire(RecordMap(simplified.map { c => (c.name, Clock()) }:_*)) - for (parameter <- simplified) { - val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(parameter)).get - outputWire(parameter.name).get := cbClockField - } - _clockRecord = Some(outputWire) + _ratRefName = Some(baseClockName) + for (clock <- allClocks) { + val clkWire = Wire(new Clock) + _ratClockMap(clock.name) = (clock, clkWire) + ratClockRecordMapWire(clock.name).get := clkWire + } + + ratClockRecordMapWire + } + + /** + * Connect all clocks requested to ClockBridge + */ + def instantiateFireSimDividerPLL: Unit = { + // Simplify the RationalClocks ratio's + val refRatClock = _ratClockMap.find(_._1 == _ratRefName.get).get._2._1 + val simpleRatClocks = _ratClockMap.map { t => + val ratClock = t._2._1 + ratClock.copy( + multiplier = ratClock.multiplier * refRatClock.divisor, + divisor = ratClock.divisor * refRatClock.multiplier).simplify + } + + // Determine all the clock dividers (harness + rational clocks) + // Note: Requires that the BuildTop reference frequency is requested with proper freq. + val refRatClockFreq = _harnessClockMap.find(_._1 == _ratRefName.get).get._2._1 + val refRatSinkParams = ClockSinkParameters(take=Some(ClockParameters(freqMHz=refRatClockFreq / (1000 * 1000))),name=Some(_ratRefName.get)) + val harSinkParams = _harnessClockMap.map { case (name, (freq, bundle)) => + ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq / (1000 * 1000))),name=Some(name)) + }.toSeq + val allSinkParams = harSinkParams :+ refRatSinkParams + + // Use PLL config to determine overall div's + val pllConfig = new SimplePllConfiguration("firesimOverallClockBridge", allSinkParams) + pllConfig.emitSummaries + + // Adjust all BuildTop RationalClocks with the div determined by the PLL + val refRatDiv = pllConfig.sinkDividerMap(refRatSinkParams) + val adjRefRatClocks = simpleRatClocks.map { clock => + clock.copy(divisor = clock.divisor * refRatDiv).simplify + } + + // Convert harness clocks to RationalClocks + val harRatClocks = harSinkParams.map { case ClockSinkParameters(_, _, _, _, clkParamsOpt, nameOpt) => + RationalClock(nameOpt.get, 1, pllConfig.referenceFreqMHz.toInt / clkParamsOpt.get.freqMHz.toInt) + } + + val allAdjRatClks = adjRefRatClocks ++ harRatClocks + + // Removes clocks that have the same frequency before instantiating the + // clock bridge to avoid unnecessary BUFGCE use. + val allDistinctRatClocks = allAdjRatClks.foldLeft(Seq(RationalClock(pllConfig.referenceSinkParams.name.get, 1, 1))) { + case (list, candidate) => if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate + } + + val clockBridge = Module(new RationalClockBridge(allDistinctRatClocks)) + val cbVecTuples = allDistinctRatClocks.zip(clockBridge.io.clocks) + + // Connect all clocks (harness + BuildTop clocks) + for (clock <- allAdjRatClks) { + val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(clock)).get + _ratClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField } + _harnessClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField } } - getClockRecord } } @@ -117,16 +179,19 @@ class WithFireSimSimpleClocks extends Config((site, here, up) => { clockBundle.reset := reset } - val pllConfig = new SimplePllConfiguration("FireSim RationalClockBridge", clockGroupEdge.sink.members) + val pllConfig = new SimplePllConfiguration("firesimBuildTopClockGenerator", clockGroupEdge.sink.members) pllConfig.emitSummaries val rationalClockSpecs = for ((sinkP, division) <- pllConfig.sinkDividerMap) yield { RationalClock(sinkP.name.get, 1, division) } + // Set the reference frequency used + chiptop.refClockFreqMHz = Some(pllConfig.referenceFreqMHz) + chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => { reset := th.harnessReset input_clocks := p(ClockBridgeInstantiatorKey) - .getClockRecordOrInstantiate(rationalClockSpecs.toSeq, p(FireSimBaseClockNameKey)) + .getClockRecordMap(rationalClockSpecs.toSeq, p(FireSimBaseClockNameKey)) Nil }) } } @@ -140,6 +205,8 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B } def success = { require(false, "success should not be used in Firesim"); false.B } + var btFreqMHz: Option[Double] = None + // Instantiate multiple instances of the DUT to implement supernode for (i <- 0 until p(NumNodes)) { // It's not a RC bump without some hacks... @@ -150,7 +217,15 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna val lazyModule = LazyModule(p(BuildTop)(p.alterPartial({ case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy }))) - val module = Module(lazyModule.module) + withClockAndReset(harnessClock, harnessReset) { + val module = Module(lazyModule.module) + } + + btFreqMHz = Some(lazyModule match { + case d: HasReferenceClockFreq => d.refClockFreqMHz.getOrElse(p(DefaultClockFrequencyKey)) + case _ => p(DefaultClockFrequencyKey) + }) + lazyModule match { case d: HasTestHarnessFunctions => require(d.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset") d.harnessFunctions.foreach(_(this)) @@ -160,5 +235,8 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna } NodeIdx.increment() } - harnessClock := p(ClockBridgeInstantiatorKey).getClockRecord("implicit_clock").get + + harnessClock := p(ClockBridgeInstantiatorKey).getClock(p(FireSimBaseClockNameKey), btFreqMHz.get * (1000 * 1000)) + + p(ClockBridgeInstantiatorKey).instantiateFireSimDividerPLL } diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala index a7626f32..6be9cc21 100644 --- a/generators/firechip/src/main/scala/TargetConfigs.scala +++ b/generators/firechip/src/main/scala/TargetConfigs.scala @@ -59,7 +59,7 @@ class WithNIC extends icenet.WithIceNIC(inBufFlits = 8192, ctrlQueueDepth = 64) class WithNVDLALarge extends nvidia.blocks.dla.WithNVDLA("large") class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small") -class WithFireSimConfigTweaksWithoutClocking extends Config( +class WithFireSimDesignTweaks extends Config( // Required: Bake in the default FASED memory model new WithDefaultMemModel ++ // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset @@ -96,7 +96,7 @@ class WithFireSimConfigTweaks extends Config( new chipyard.config.WithAsynchrousMemoryBusCrossing ++ new testchipip.WithAsynchronousSerialSlaveCrossing ++ // Tweaks that are independent from multi-clock - new WithFireSimConfigTweaksWithoutClocking + new WithFireSimDesignTweaks ) /******************************************************************************* @@ -207,8 +207,9 @@ class FireSimMulticlockRocketConfig extends Config( class FireSimMulticlockAXIOverSerialConfig extends Config( new WithAXIOverSerialTLCombinedBridges ++ // use combined bridge to connect to axi mem over serial new WithDefaultFireSimBridges ++ + new testchipip.WithBlockDevice(false) ++ // disable blockdev new WithDefaultMemModel ++ - new WithFireSimConfigTweaksWithoutClocking ++ // don't inherit firesim clocking + new WithFireSimDesignTweaks ++ // don't inherit firesim clocking new chipyard.MulticlockAXIOverSerialConfig ) diff --git a/generators/testchipip b/generators/testchipip index 927709c0..03c4ac48 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit 927709c09e404c64108bb909f894e35c9a63396c +Subproject commit 03c4ac486209253aaa20a702842fba511cfeac04