opae rtl fixes

hw/opae/Makefile

@@ -15,7 +15,7 @@ setup-ase: $(ASE_BUILD_DIR)/Makefile
 setup-fpga: $(FPGA_BUILD_DIR)/build/dcp.qpf
 
 $(ASE_BUILD_DIR)/Makefile:
-	afu_sim_setup --s sources.txt $(ASE_BUILD_DIR)
+	afu_sim_setup -s sources.txt $(ASE_BUILD_DIR)
 
 $(FPGA_BUILD_DIR)/build/dcp.qpf:
 	afu_synth_setup -s sources.txt $(FPGA_BUILD_DIR)

hw/opae/README

@@ -7,7 +7,7 @@ source /export/fpga/bin/setup-fpga-env fpga-pac-a10
 ## Vortex Run commands ##
 #########################
 ## Synthesis
-cd /driver/hw/
+cd /driver/hw/opae
 # Configure a Quartus build area
 afu_synth_setup -s sources.txt build_fpga
 cd build_fpga
@@ -47,6 +47,9 @@ source /export/fpga/bin/setup-fpga-env fpga-pac-a10
 # Acquire a sever node for running ASE simulations
 qsub-sim
 
+# test
+./run_ase.sh ../../driver/tests/basic/basic
+
 # modify "vsim_run.tcl" to dump VCD trace
 vcd file vortex.vcd
 vcd add -r /*/Vortex/hw/rtl/* 
@@ -61,9 +64,6 @@ tar -zxvf /mnt/c/Users/Blaise/Downloads/vortex.vcd.tar.gz
 # launch Gtkwave
 gtkwave ./build_ase/work/vortex.vcd &
 
-# test
-./run_ase.sh ../../driver/tests/basic/basic
-
 # kill process by Users
 ps -u tinebp
 kill -9 <pid>

hw/opae/run_ase.sh

@@ -10,12 +10,14 @@ export ASE_WORKDIR=$SCRIPT_DIR/build_ase/work
 shift 1
 
 # cleanup incomplete runs
-rm -rf $ASE_WORKDIR/.app_lock.pid $ASE_WORKDIR/.ase_ready.pid
+rm -f $ASE_WORKDIR/.app_lock.pid 
+rm -f $ASE_WORKDIR/.ase_ready.pid
+rm -f $SCRIPT_DIR/build_ase/nohup.out
 
 # Start Simulator in background
 pushd $SCRIPT_DIR/build_ase 
-echo "  [DBG]  starting ASE simnulator"
-nohup make sim  & 
+echo "  [DBG]  starting ASE simnulator (stdout saved to '$SCRIPT_DIR/build_ase/nohup.out')"
+nohup make sim & 
 popd
 
 # Wait for simulator readiness

hw/opae/sources.txt

@@ -2,6 +2,9 @@ vortex_afu.json
 
 +define+GLOBAL_BLOCK_SIZE=64
 
+#+define+NUM_CORES=2
+#+define+DL2_ENABLE=0
+
 +incdir+.
 +incdir+../rtl
 +incdir+../rtl/interfaces
@@ -23,9 +26,10 @@ vortex_afu.json
 ../rtl/cache/VX_cache_req_queue.v
 ../rtl/cache/VX_cache_miss_resrv.v
 ../rtl/cache/VX_fill_invalidator.v
-../rtl/cache/VX_snp_fwd_arb.v
+../rtl/cache/VX_snp_rsp_arb.v
 ../rtl/cache/VX_tag_data_access.v
 ../rtl/cache/VX_tag_data_structure.v
+../rtl/cache/VX_snp_forwarder.v
 ../rtl/cache/VX_prefetcher.v
 
 ../rtl/interfaces/VX_branch_rsp_if.v
@@ -34,6 +38,7 @@ vortex_afu.json
 ../rtl/interfaces/VX_cache_dram_req_if.v
 ../rtl/interfaces/VX_cache_dram_rsp_if.v
 ../rtl/interfaces/VX_cache_snp_req_if.v
+../rtl/interfaces/VX_cache_snp_rsp_if.v
 ../rtl/interfaces/VX_csr_req_if.v
 ../rtl/interfaces/VX_exec_unit_req_if.v
 ../rtl/interfaces/VX_frE_to_bckE_req_if.v

hw/opae/vortex_afu.sv

@@ -40,9 +40,6 @@ localparam AVS_RD_QUEUE_SIZE  = 16;
 localparam CCI_RD_WINDOW_SIZE = 8;
 localparam CCI_RD_QUEUE_SIZE  = 2 * CCI_RD_WINDOW_SIZE;
 
-localparam VX_SNOOP_DELAY     = 1000;
-localparam VX_SNOOP_LEVELS    = 2;
-
 localparam AFU_ID_L           = 16'h0002;      // AFU ID Lower
 localparam AFU_ID_H           = 16'h0004;      // AFU ID Higher 
 
@@ -89,8 +86,13 @@ logic vx_dram_rsp_ready;
 
 logic vx_snp_req_valid;
 logic [DRAM_ADDR_WIDTH-1:0] vx_snp_req_addr;
+logic [0:0] vx_snp_req_tag;
 logic vx_snp_req_ready;
 
+logic vx_snp_rsp_valid;
+logic [0:0] vx_snp_rsp_addr;
+logic vx_snp_rsp_ready;
+
 logic vx_busy;
 
 // AVS Queues /////////////////////////////////////////////////////////////////
@@ -207,9 +209,8 @@ end
 logic [DRAM_ADDR_WIDTH-1:0] cci_write_ctr;
 logic [DRAM_ADDR_WIDTH-1:0] avs_read_ctr;
 logic [DRAM_ADDR_WIDTH-1:0] avs_write_ctr;
-logic [DRAM_ADDR_WIDTH-1:0] snp_req_ctr;
-logic [9:0] snp_req_delay;
-logic       vx_reset;
+logic                       vx_reset;
+logic                       snp_rsp_done;
 
 always_ff @(posedge clk) 
 begin
@@ -267,7 +268,7 @@ begin
       end
 
       STATE_CLFLUSH: begin
-        if (snp_req_delay >= VX_SNOOP_DELAY) begin
+        if (snp_rsp_done) begin
           state <= STATE_IDLE;
         end
       end
@@ -572,33 +573,48 @@ end
 
 // Vortex cache snooping //////////////////////////////////////////////////////
 
+logic [DRAM_ADDR_WIDTH-1:0] snp_req_ctr;
+logic [DRAM_ADDR_WIDTH-1:0] snp_rsp_ctr;
+
+always_comb 
+begin
+  snp_rsp_done = (snp_rsp_ctr >= csr_data_size);
+end
+
 always_ff @(posedge clk) 
 begin
   if (SoftReset) begin
     vx_snp_req_valid <= 0;
+    vx_snp_req_tag   <= 0;
+    vx_snp_rsp_ready <= 0;
     snp_req_ctr      <= 0;
-    snp_req_delay    <= 0;
+    snp_rsp_ctr      <= 0;
   end
   else begin
     if (STATE_IDLE == state) begin
-      snp_req_ctr   <= 0;
-      snp_req_delay <= 0;
+      snp_req_ctr      <= 0;
+      snp_rsp_ctr      <= 0;
+      vx_snp_rsp_ready <= 0;
     end
 
     vx_snp_req_valid <= 0;
+    vx_snp_rsp_ready <= 0;
 
     if ((STATE_CLFLUSH == state)
      && (snp_req_ctr < csr_data_size)
      && vx_snp_req_ready)
     begin
       vx_snp_req_addr  <= csr_mem_addr + snp_req_ctr;
-      vx_snp_req_valid <= 1;
       snp_req_ctr      <= snp_req_ctr + 1;
+      vx_snp_req_valid <= 1;
+      vx_snp_rsp_ready <= 1;
     end
 
-    if (snp_req_ctr == csr_data_size) begin 
-      snp_req_delay <= snp_req_delay + 1;
-    end
+    if ((STATE_CLFLUSH == state) 
+     && (snp_rsp_ctr < csr_data_size)
+     && vx_snp_rsp_valid) begin
+       snp_rsp_ctr <= snp_rsp_ctr + 1;
+    end   
   end
 end
 
@@ -622,11 +638,17 @@ Vortex_Socket #() vx_socket (
   .dram_rsp_tag     (vx_dram_rsp_tag),
   .dram_rsp_ready   (vx_dram_rsp_ready),
 
-  // Cache snooping
+  // Snoop request
   .snp_req_valid 	  (vx_snp_req_valid),
   .snp_req_addr     (vx_snp_req_addr),
+  .snp_req_tag      (vx_snp_req_tag),
   .snp_req_ready    (vx_snp_req_ready),
 
+  // Snoop response
+  .snp_rsp_valid 	  (vx_snp_rsp_valid),
+  .snp_rsp_tag      (vx_snp_rsp_tag),
+  .snp_rsp_ready    (vx_snp_rsp_ready),
+
   // I/O request
   .io_req_read      (),
   .io_req_write     (),