From 86deaa8e076eef42e48e6f90cb526064d67b1813 Mon Sep 17 00:00:00 2001
From: Hansung Kim <hansung_kim@berkeley.edu>
Date: Wed, 12 Jun 2024 09:46:51 -0700
Subject: [PATCH] Give some slack time for other cores to finish

---
 hw/rtl/VX_core_wrapper.sv  | 33 ++++++++++++++++++++++-----------
 hw/rtl/core/VX_schedule.sv |  2 ++
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/hw/rtl/VX_core_wrapper.sv b/hw/rtl/VX_core_wrapper.sv
index 626a3cb1..2d516cf4 100644
--- a/hw/rtl/VX_core_wrapper.sv
+++ b/hw/rtl/VX_core_wrapper.sv
@@ -495,20 +495,30 @@ module Vortex import VX_gpu_pkg::*; #(
     //     .busy(busy)
     // );
 
+    logic [31:0] finish_counter;
+
+    always @(posedge clock) begin
+      if (reset) begin
+        finish_counter <= 32'd0;
+      end else begin
+        if (finished) begin
+          finish_counter <= finish_counter + 32'd1;
+        end
+      end
+    end
+
+    // give slack for other cores to finish
+    wire all_cores_finished = (finish_counter > 32'd1000);
+
+`ifdef SIMULATION
     always @(posedge clock) begin
         if (!reset) begin
+            if (all_cores_finished) begin
+                $display("simulation has probably ended. exiting");
+                $finish();
+            end
             if (finished) begin
-                `ifdef SIMULATION
-                    $display("---------------- core%2d has no more active warps ----------------", CORE_ID);
-                    $display("simulation has ended. exiting");
-                    $finish();
-                `endif
-                // `ifdef SIMULATION
-                // if ($time >= 60000) begin
-                //   $display("simulation has probably ended. exiting");
-                //   @(posedge clock) $finish();
-                // end
-                // `endif
+                $display("---------------- core%2d has no more active warps ----------------", CORE_ID);
                 // TODO: lane assumed to be 4
                 // `ifndef SYNTHESIS
                 // for (integer j = 0; j < `NUM_WARPS; j++) begin
@@ -525,6 +535,7 @@ module Vortex import VX_gpu_pkg::*; #(
             end
         end
     end
+`endif
 
 endmodule : Vortex
 
diff --git a/hw/rtl/core/VX_schedule.sv b/hw/rtl/core/VX_schedule.sv
index 53a11ffb..1b3cd5ee 100644
--- a/hw/rtl/core/VX_schedule.sv
+++ b/hw/rtl/core/VX_schedule.sv
@@ -280,6 +280,8 @@ module VX_schedule import VX_gpu_pkg::*; #(
     assign gbar_bus_if.req_valid   = gbar_req_valid;
     assign gbar_bus_if.req_id      = gbar_req_id;
     assign gbar_bus_if.req_size_m1 = gbar_req_size_m1;
+    // NOTE(hansung): since CORE_ID is global across multiple clusters, we
+    // need the modulo to get the per-cluster local core id
     assign gbar_bus_if.req_core_id = `NC_WIDTH'(CORE_ID % `NUM_CORES);
 `endif