diff --git a/src/main/scala/radiance/subsystem/Configs.scala b/src/main/scala/radiance/subsystem/Configs.scala index 971f4d6..83f8007 100644 --- a/src/main/scala/radiance/subsystem/Configs.scala +++ b/src/main/scala/radiance/subsystem/Configs.scala @@ -38,7 +38,8 @@ class WithRadianceCores( ) extends Config((site, _, up) => { case TilesLocated(`location`) => { val prev = up(TilesLocated(`location`)) - val idOffset = prev.size + val idOffset = up(NumTiles) + val coreIdOffset = up(NumRadianceCores) val vortex = RadianceTileParams( core = VortexCoreParams(fpu = None), btb = None, @@ -63,10 +64,15 @@ class WithRadianceCores( nTLBSuperpages = 1, blockBytes = site(CacheBlockBytes)))) List.tabulate(n)(i => RadianceTileAttachParams( - vortex.copy(tileId = i + idOffset), + vortex.copy( + tileId = i + idOffset, + coreId = i + coreIdOffset, + ), crossing )) ++ prev } + case NumTiles => up(NumTiles) + n + case NumRadianceCores => up(NumRadianceCores) + n }) { def this(n: Int, location: HierarchicalLocation = InSubsystem, useVxCache: Boolean = false) = this(n, location, RocketCrossingParams( master = HierarchicalElementMasterPortParams.locationDefault(location), @@ -83,8 +89,9 @@ class WithRadianceGemmini(location: HierarchicalLocation, dim: Int, accSizeInKB: Int, tileSize: Int) extends Config((site, _, up) => { case TilesLocated(`location`) => { val prev = up(TilesLocated(`location`)) - val idOffset = prev.size + val idOffset = up(NumTiles) if (idOffset == 0) { + // FIXME: this doesn't work for multiple clusters when idOffset may not be 0 println("******WARNING****** gemmini tile id is 0! radiance tiles in the same cluster needs to be before gemmini") } val numPrevGemminis = prev.map(_.tileParams).map { @@ -121,6 +128,7 @@ class WithRadianceGemmini(location: HierarchicalLocation, crossing )) ++ prev } + case NumTiles => up(NumTiles) + 1 }) { def this(location: HierarchicalLocation = InSubsystem, dim: Int, accSizeInKB: Int, tileSize: Int) = this(location, RocketCrossingParams( @@ -169,7 +177,7 @@ class WithFuzzerCores( ) extends Config((site, _, up) => { case TilesLocated(InSubsystem) => { val prev = up(TilesLocated(InSubsystem)) - val idOffset = prev.size + val idOffset = up(NumTiles) val fuzzer = FuzzerTileParams( core = VortexCoreParams(fpu = None), useVxCache = useVxCache) @@ -178,6 +186,8 @@ class WithFuzzerCores( RocketCrossingParams() )) ++ prev } + case NumTiles => up(NumTiles) + 1 + case NumRadianceCores => up(NumRadianceCores) + 1 }) class WithRadianceCluster( @@ -285,7 +295,7 @@ class WithNCustomSmallRocketCores( ) extends Config((site, here, up) => { case TilesLocated(InSubsystem) => { val prev = up(TilesLocated(InSubsystem)) - val idOffset = overrideIdOffset.getOrElse(prev.size) + val idOffset = up(NumTiles) val med = RocketTileParams( core = RocketCoreParams(fpu = None), btb = None, @@ -313,6 +323,7 @@ class WithNCustomSmallRocketCores( crossing )) ++ prev } + case NumTiles => up(NumTiles) + n }) class WithExtGPUMem(address: BigInt = BigInt("0x100000000", 16), diff --git a/src/main/scala/radiance/tile/Barrier.scala b/src/main/scala/radiance/tile/Barrier.scala index 269a62c..02ae4fd 100644 --- a/src/main/scala/radiance/tile/Barrier.scala +++ b/src/main/scala/radiance/tile/Barrier.scala @@ -73,7 +73,7 @@ class BarrierSynchronizer( ) extends Module { val numBarriers = 1 << param.barrierIdBits val numCores = 1 << param.numCoreBits - println(s"numBarriers: ${numBarriers}, numCores: ${numCores}") + println(s"======== numBarriers: ${numBarriers}, numCores: ${numCores}") val io = IO(new Bundle { val reqs = Vec(numCores, Flipped(Decoupled(new BarrierRequestBits(param)))) diff --git a/src/main/scala/radiance/tile/RadianceCluster.scala b/src/main/scala/radiance/tile/RadianceCluster.scala index c75f4f3..9c60854 100644 --- a/src/main/scala/radiance/tile/RadianceCluster.scala +++ b/src/main/scala/radiance/tile/RadianceCluster.scala @@ -34,12 +34,6 @@ class RadianceCluster ( crossing: ClockCrossingType, lookup: LookupByClusterIdImpl )(implicit p: Parameters) extends Cluster(thisClusterParams, crossing, lookup) { - // cluster-local bus, used for shared memory traffic that never leaves the - // confines of a cluster - val clbus = tlBusWrapperLocationMap(CLBUS(clusterId)) - - clbus.clockGroupNode := allClockGroupsNode - // Instantiate cluster-local shared memory scratchpad // // Instantiate the same number of banks as there are lanes. @@ -53,7 +47,7 @@ class RadianceCluster ( val radianceTiles = leafTiles.values.filter(_.isInstanceOf[RadianceTile]).toSeq.asInstanceOf[Seq[RadianceTile]] - val numCores = leafTiles.size - gemminis.size + val numCoresInCluster = leafTiles.size - gemminis.size // ************************************** // ______ _________ ___ @@ -331,7 +325,7 @@ class RadianceCluster ( // // ******************************************************* - val radianceAccSlaveNodes = Seq.fill(numCores)(AccSlaveNode()) + val radianceAccSlaveNodes = Seq.fill(numCoresInCluster)(AccSlaveNode()) (radianceAccSlaveNodes zip radianceTiles).foreach { case (a, r) => a := r.accMasterNode } val gemminiAccMasterNodes = gemminiTiles.map { tile => val masterNode = AccMasterNode() @@ -342,8 +336,8 @@ class RadianceCluster ( val traceTLNode = TLAdapterNode(clientFn = c => c, managerFn = m => m) // printf and perf counter buffer - TLRAM(AddressSet(smem_key.address + smem_size, numCores * 0x200 - 1)) := traceTLNode := - TLBuffer() := TLFragmenter(4, 4) := clbus.outwardNode + TLRAM(AddressSet(smem_key.address + smem_size, numCoresInCluster * 0x200 - 1)) := + traceTLNode := TLBuffer() := TLFragmenter(4, 4) := clbus.outwardNode p(RadianceFrameBufferKey).foreach { key => val fb = LazyModule(new FrameBuffer(key.baseAddress, key.width, key.size, key.validAddress, key.fbName)) @@ -351,7 +345,7 @@ class RadianceCluster ( } // Diplomacy sink nodes for cluster-wide barrier sync signal - val barrierSlaveNode = BarrierSlaveNode(numCores) + val barrierSlaveNode = BarrierSlaveNode(numCoresInCluster) // HACK: This is a workaround of the CanAttachTile bus connecting API that // works by downcasting tile and directly accessing the node inside that is @@ -381,7 +375,6 @@ class RadianceClusterModuleImp(outer: RadianceCluster) extends ClusterModuleImp( // cores are configured to have the same barrier id range. While true, might // be better to actually assert this val barrierParam = outer.barrierSlaveNode.in.head._2 - println(s"======= barrierParam: ${barrierParam}") val synchronizer = Module(new BarrierSynchronizer(barrierParam)) (synchronizer.io.reqs zip outer.barrierSlaveNode.in).foreach { case (req, (b, _)) => req <> b.req @@ -542,6 +535,4 @@ class RadianceClusterModuleImp(outer: RadianceCluster) extends ClusterModuleImp( } makeSmemBanks() - - println(s"======== barrierSlaveNode: ${outer.barrierSlaveNode.in(0)._2.barrierIdBits}") } diff --git a/src/main/scala/radiance/tile/RadianceTile.scala b/src/main/scala/radiance/tile/RadianceTile.scala index 2a6325c..a05a045 100644 --- a/src/main/scala/radiance/tile/RadianceTile.scala +++ b/src/main/scala/radiance/tile/RadianceTile.scala @@ -21,6 +21,10 @@ import org.chipsalliance.cde.config._ import radiance.memory._ import radiance.subsystem.{GPUMemParams, GPUMemory, RadianceSimArgs} +/** For determining radiance core id. This may be different from + * RadianceTileParams.coreId, when a cluster contains non-core tiles */ +case object NumRadianceCores extends Field[Int](0) + case class RadianceTileParams( core: VortexCoreParams = VortexCoreParams(), useVxCache: Boolean = false, @@ -30,6 +34,7 @@ case class RadianceTileParams( dataScratchpadBytes: Int = 0, name: Option[String] = Some("radiance_tile"), tileId: Int = 0, + coreId: Int = 0, beuAddr: Option[BigInt] = None, blockerCtrlAddr: Option[BigInt] = None, clockSinkParams: ClockSinkParameters = ClockSinkParameters(), @@ -210,7 +215,7 @@ class RadianceTile private ( clients = Seq( TLMasterParameters.v1( sourceId = IdRange(0, 1 << imemSourceWidth), - name = s"Vortex Core ${radianceParams.tileId} I-Mem $i", + name = s"Vortex Core ${radianceParams.coreId} I-Mem $i", requestFifo = true, supportsProbe = TransferSizes(1, lazyCoreParamsView.coreDataBytes), @@ -229,7 +234,7 @@ class RadianceTile private ( clients = Seq( TLMasterParameters.v1( sourceId = IdRange(0, 1 << dmemSourceWidth), - name = s"Vortex Core ${radianceParams.tileId} D-Mem Lane $i", + name = s"Vortex Core ${radianceParams.coreId} D-Mem Lane $i", requestFifo = true, supportsProbe = TransferSizes(1, lazyCoreParamsView.coreDataBytes), @@ -252,7 +257,7 @@ class RadianceTile private ( clients = Seq( TLMasterParameters.v1( sourceId = IdRange(0, 1 << smemSourceWidth), - name = s"Vortex Core ${radianceParams.tileId} SharedMem Lane $i", + name = s"Vortex Core ${radianceParams.coreId} SharedMem Lane $i", requestFifo = true, supportsProbe = TransferSizes(1, lazyCoreParamsView.coreDataBytes), @@ -285,7 +290,7 @@ class RadianceTile private ( TLMasterParameters.v1( // FIXME: need to also respect imemSourceWidth sourceId = IdRange(0, 1 << dmemSourceWidth), - name = s"Vortex Core ${radianceParams.tileId} Mem Interface", + name = s"Vortex Core ${radianceParams.coreId} Mem Interface", requestFifo = true, supportsProbe = TransferSizes(16, 16), // FIXME: hardcoded supportsGet = TransferSizes(16, 16), @@ -532,7 +537,7 @@ class RadianceTileModuleImp(outer: RadianceTile) core.io.imem.get(0).d <> imemTLAdapter.io.inResp performanceCounters(Seq(imemTLAdapter.io.inReq), Seq(imemTLAdapter.io.inResp), - desc = s"core${outer.tileId}-imem") + desc = s"core${outer.radianceParams.coreId}-imem") // now connect TL adapter downstream ports to the tile egress ports outer.imemNodes(0).out(0)._1.a <> imemTLAdapter.io.outReq @@ -641,7 +646,7 @@ class RadianceTileModuleImp(outer: RadianceTile) } performanceCounters(dmemTLAdapters.map(_.io.inReq), dmemTLAdapters.map(_.io.inResp), - desc = s"core${outer.tileId}-dmem") + desc = s"core${outer.radianceParams.coreId}-dmem") // now connect TL adapter downstream ports to the tile egress ports (dmemTLAdapters zip dmemTLBundles) foreach { case (tlAdapter, tlOut) => @@ -702,7 +707,7 @@ class RadianceTileModuleImp(outer: RadianceTile) } performanceCounters(smemTLAdapters.map(_.io.inReq), smemTLAdapters.map(_.io.inResp), - desc = s"core${outer.tileId}-smem") + desc = s"core${outer.radianceParams.coreId}-smem") // now connect TL adapter downstream ports to the tile egress ports (smemTLAdapters zip smemTLBundles) foreach { case (tlAdapter, tlOut) => diff --git a/src/main/scala/radiance/tile/VortexCore.scala b/src/main/scala/radiance/tile/VortexCore.scala index dab3ec0..090e9ff 100644 --- a/src/main/scala/radiance/tile/VortexCore.scala +++ b/src/main/scala/radiance/tile/VortexCore.scala @@ -117,11 +117,10 @@ class VortexBundle(tile: RadianceTile)(implicit p: Parameters) extends CoreBundl class Vortex(tile: RadianceTile)(implicit p: Parameters) extends BlackBox( - // Each Vortex core gets tied-off tileId of 0, 1, 2, 3, ... - // The actual MHARTID read by the program is different by warp, not core; - // see VX_csr_data that implements the read logic for CSR_MHARTID/GWID. + // Each Vortex core gets tied-off core id of 0, 1, 2, 3, which is global + // across multiple clusters. Map( - "CORE_ID" -> tile.tileParams.tileId, + "CORE_ID" -> tile.radianceParams.coreId, // TODO: can we get this as a parameter? "BOOTROM_HANG100" -> 0x10100, "NUM_THREADS" -> tile.numLsuLanes