Separate out core id from tile id in TileParams

Create a new config key to distinguish number of cores from number of
total tiles (which can be different when there are Gemmini tiles).

It is important to give contiguous IDs for Vortex cores for the
cluter-wide barrier to work.

src/main/scala/radiance/subsystem/Configs.scala

@@ -44,6 +44,7 @@ class WithRadianceCores(
   case TilesLocated(`location`) => {
     val prev = up(TilesLocated(`location`))
     val idOffset = up(NumTiles)
+    val coreIdOffset = up(NumRadianceCores)
     val vortex = RadianceTileParams(
       core = VortexCoreParams(fpu = None),
       btb = None,
@@ -68,11 +69,15 @@ class WithRadianceCores(
         nTLBSuperpages = 1,
         blockBytes = site(CacheBlockBytes))))
     List.tabulate(n)(i => RadianceTileAttachParams(
-      vortex.copy(tileId = i + idOffset),
+      vortex.copy(
+        tileId = i + idOffset,
+        coreId = i + coreIdOffset,
+      ),
       crossing
     )) ++ prev
   }
   case NumTiles => up(NumTiles) + n
+  case NumRadianceCores => up(NumRadianceCores) + n
 }) {
   def this(n: Int, location: HierarchicalLocation = InSubsystem, useVxCache: Boolean = false) = this(n, location, RocketCrossingParams(
     master = HierarchicalElementMasterPortParams.locationDefault(location),
@@ -127,6 +132,8 @@ class WithRadianceGemmini(location: HierarchicalLocation,
     ))
   }
   case NumTiles => up(NumTiles) + 1
+  // don't increment core id for Gemmini tiles
+  case NumRadianceCores => up(NumRadianceCores)
 }) {
   def this(location: HierarchicalLocation = InSubsystem, dim: Int, accSizeInKB: Int, tileSize: Int) =
     this(location, RocketCrossingParams(
@@ -185,6 +192,7 @@ class WithFuzzerCores(
     )) ++ prev
   }
   case NumTiles => up(NumTiles) + 1
+  case NumRadianceCores => up(NumRadianceCores) + 1
 })
 
 class WithRadianceCluster(

src/main/scala/radiance/tile/Barrier.scala

@@ -73,7 +73,7 @@ class BarrierSynchronizer(
 ) extends Module {
   val numBarriers = 1 << param.barrierIdBits
   val numCores = 1 << param.numCoreBits
-  println(s"numBarriers: ${numBarriers}, numCores: ${numCores}")
+  println(s"======== numBarriers: ${numBarriers}, numCores: ${numCores}")
 
   val io = IO(new Bundle {
     val reqs = Vec(numCores, Flipped(Decoupled(new BarrierRequestBits(param))))

src/main/scala/radiance/tile/RadianceCluster.scala

@@ -51,7 +51,7 @@ class RadianceCluster (
 
   val radianceTiles = leafTiles.values.filter(_.isInstanceOf[RadianceTile]).toSeq.asInstanceOf[Seq[RadianceTile]]
 
-  val numCores = leafTiles.size - gemminis.size
+  val numCoresInCluster = leafTiles.size - gemminis.size
 
   // **************************************
   //    ______  _________  ___
@@ -324,7 +324,7 @@ class RadianceCluster (
   // connect tile smem nodes to xbar, and xbar to banks
   // val smem_xbar = TLXbar()
 
-  val radianceAccSlaveNodes = Seq.fill(numCores)(AccSlaveNode())
+  val radianceAccSlaveNodes = Seq.fill(numCoresInCluster)(AccSlaveNode())
   (radianceAccSlaveNodes zip radianceTiles).foreach { case (a, r) => a := r.accMasterNode }
   val gemminiAccMasterNode = AccMasterNode()
   gemminiTile.accSlaveNode := gemminiAccMasterNode
@@ -332,8 +332,8 @@ class RadianceCluster (
 
   val traceTLNode = TLAdapterNode(clientFn = c => c, managerFn = m => m)
   // printf and perf counter buffer
-  TLRAM(AddressSet(smem_key.address + smem_size, numCores * 0x200 - 1)) := traceTLNode :=
-    TLBuffer() := TLFragmenter(4, 4) := clbus.outwardNode
+  TLRAM(AddressSet(smem_key.address + smem_size, numCoresInCluster * 0x200 - 1)) :=
+    traceTLNode := TLBuffer() := TLFragmenter(4, 4) := clbus.outwardNode
 
   p(RadianceFrameBufferKey).foreach { key =>
     val fb = LazyModule(new FrameBuffer(key.baseAddress, key.width, key.size, key.validAddress, key.fbName))
@@ -341,7 +341,7 @@ class RadianceCluster (
   }
 
   // Diplomacy sink nodes for cluster-wide barrier sync signal
-  val barrierSlaveNode = BarrierSlaveNode(numCores)
+  val barrierSlaveNode = BarrierSlaveNode(numCoresInCluster)
 
   // HACK: This is a workaround of the CanAttachTile bus connecting API that
   // works by downcasting tile and directly accessing the node inside that is
@@ -371,7 +371,6 @@ class RadianceClusterModuleImp(outer: RadianceCluster) extends ClusterModuleImp(
   // cores are configured to have the same barrier id range.  While true, might
   // be better to actually assert this
   val barrierParam = outer.barrierSlaveNode.in.head._2
-  println(s"======= barrierParam: ${barrierParam}")
   val synchronizer = Module(new BarrierSynchronizer(barrierParam))
   (synchronizer.io.reqs zip outer.barrierSlaveNode.in).foreach { case (req, (b, _)) =>
     req <> b.req
@@ -528,6 +527,4 @@ class RadianceClusterModuleImp(outer: RadianceCluster) extends ClusterModuleImp(
   }
 
   makeSmemBanks()
-
-  println(s"======== barrierSlaveNode: ${outer.barrierSlaveNode.in(0)._2.barrierIdBits}")
 }

src/main/scala/radiance/tile/RadianceTile.scala

@@ -21,6 +21,10 @@ import org.chipsalliance.cde.config._
 import radiance.memory._
 import radiance.subsystem.{GPUMemParams, GPUMemory, RadianceSimArgs}
 
+/** For determining radiance core id.  This may be different from
+ *  RadianceTileParams.coreId, when a cluster contains non-core tiles */
+case object NumRadianceCores extends Field[Int](0)
+
 case class RadianceTileParams(
     core: VortexCoreParams = VortexCoreParams(),
     useVxCache: Boolean = false,
@@ -30,6 +34,7 @@ case class RadianceTileParams(
     dataScratchpadBytes: Int = 0,
     name: Option[String] = Some("radiance_tile"),
     tileId: Int = 0,
+    coreId: Int = 0,
     beuAddr: Option[BigInt] = None,
     blockerCtrlAddr: Option[BigInt] = None,
     clockSinkParams: ClockSinkParameters = ClockSinkParameters(),
@@ -210,7 +215,7 @@ class RadianceTile private (
           clients = Seq(
             TLMasterParameters.v1(
               sourceId = IdRange(0, 1 << imemSourceWidth),
-              name = s"Vortex Core ${radianceParams.tileId} I-Mem $i",
+              name = s"Vortex Core ${radianceParams.coreId} I-Mem $i",
               requestFifo = true,
               supportsProbe =
                 TransferSizes(1, lazyCoreParamsView.coreDataBytes),
@@ -229,7 +234,7 @@ class RadianceTile private (
           clients = Seq(
             TLMasterParameters.v1(
               sourceId = IdRange(0, 1 << dmemSourceWidth),
-              name = s"Vortex Core ${radianceParams.tileId} D-Mem Lane $i",
+              name = s"Vortex Core ${radianceParams.coreId} D-Mem Lane $i",
               requestFifo = true,
               supportsProbe =
                 TransferSizes(1, lazyCoreParamsView.coreDataBytes),
@@ -252,7 +257,7 @@ class RadianceTile private (
           clients = Seq(
             TLMasterParameters.v1(
               sourceId = IdRange(0, 1 << smemSourceWidth),
-              name = s"Vortex Core ${radianceParams.tileId} SharedMem Lane $i",
+              name = s"Vortex Core ${radianceParams.coreId} SharedMem Lane $i",
               requestFifo = true,
               supportsProbe =
                 TransferSizes(1, lazyCoreParamsView.coreDataBytes),
@@ -285,7 +290,7 @@ class RadianceTile private (
           TLMasterParameters.v1(
             // FIXME: need to also respect imemSourceWidth
             sourceId = IdRange(0, 1 << dmemSourceWidth),
-            name = s"Vortex Core ${radianceParams.tileId} Mem Interface",
+            name = s"Vortex Core ${radianceParams.coreId} Mem Interface",
             requestFifo = true,
             supportsProbe = TransferSizes(16, 16), // FIXME: hardcoded
             supportsGet = TransferSizes(16, 16),
@@ -532,7 +537,7 @@ class RadianceTileModuleImp(outer: RadianceTile)
       core.io.imem.get(0).d <> imemTLAdapter.io.inResp
 
       performanceCounters(Seq(imemTLAdapter.io.inReq), Seq(imemTLAdapter.io.inResp),
-        desc = s"core${outer.tileId}-imem")
+        desc = s"core${outer.radianceParams.coreId}-imem")
 
       // now connect TL adapter downstream ports to the tile egress ports
       outer.imemNodes(0).out(0)._1.a <> imemTLAdapter.io.outReq
@@ -641,7 +646,7 @@ class RadianceTileModuleImp(outer: RadianceTile)
       }
 
       performanceCounters(dmemTLAdapters.map(_.io.inReq), dmemTLAdapters.map(_.io.inResp),
-        desc = s"core${outer.tileId}-dmem")
+        desc = s"core${outer.radianceParams.coreId}-dmem")
 
       // now connect TL adapter downstream ports to the tile egress ports
       (dmemTLAdapters zip dmemTLBundles) foreach { case (tlAdapter, tlOut) =>
@@ -702,7 +707,7 @@ class RadianceTileModuleImp(outer: RadianceTile)
       }
 
       performanceCounters(smemTLAdapters.map(_.io.inReq), smemTLAdapters.map(_.io.inResp),
-        desc = s"core${outer.tileId}-smem")
+        desc = s"core${outer.radianceParams.coreId}-smem")
 
       // now connect TL adapter downstream ports to the tile egress ports
       (smemTLAdapters zip smemTLBundles) foreach { case (tlAdapter, tlOut) =>

src/main/scala/radiance/tile/VortexCore.scala

@@ -117,11 +117,10 @@ class VortexBundle(tile: RadianceTile)(implicit p: Parameters) extends CoreBundl
 
 class Vortex(tile: RadianceTile)(implicit p: Parameters)
     extends BlackBox(
-      // Each Vortex core gets tied-off tileId of 0, 1, 2, 3, ...
-      // The actual MHARTID read by the program is different by warp, not core;
-      // see VX_csr_data that implements the read logic for CSR_MHARTID/GWID.
+      // Each Vortex core gets tied-off core id of 0, 1, 2, 3, which is global
+      // across multiple clusters.
       Map(
-        "CORE_ID" -> tile.tileParams.tileId,
+        "CORE_ID" -> tile.radianceParams.coreId,
         // TODO: can we get this as a parameter?
         "BOOTROM_HANG100" -> 0x10100,
         "NUM_THREADS" -> tile.numLsuLanes