From 16d5a8a09cc4c502f104f77067410e5912faf372 Mon Sep 17 00:00:00 2001
From: Blaise Tine <tinebp@vsl083.research.intel-research.net>
Date: Sun, 31 May 2020 14:51:42 -0700
Subject: [PATCH] opae rtl fixes

---
 driver/rtlsim/Makefile                        |   4 -
 hw/Makefile                                   |   4 -
 hw/opae/Makefile                              |   2 +-
 hw/opae/README                                |   4 +-
 hw/opae/sources.txt                           |  23 +-
 hw/opae/vortex_afu.sv                         | 651 +++++++++---------
 hw/rtl/VX_config.vh                           |   2 +-
 hw/rtl/VX_define.vh                           |  24 +-
 hw/rtl/VX_dmem_ctrl.v                         |   2 +-
 hw/rtl/VX_icache_stage.v                      |   2 +-
 hw/rtl/VX_lsu_unit.v                          |  44 +-
 hw/rtl/VX_scheduler.v                         |   1 +
 hw/rtl/Vortex.v                               |   2 +-
 hw/rtl/Vortex_Cluster.v                       |   9 +-
 hw/rtl/Vortex_Socket.v                        |  24 +-
 hw/rtl/cache/VX_cache.v                       |   1 +
 hw/rtl/cache/VX_cache_config.vh               |   2 -
 hw/rtl/cache/VX_snp_forwarder.v               |   9 +-
 hw/rtl/cache/VX_tag_data_structure.v          |  14 +-
 hw/syn/quartus/top/Makefile                   |   6 +-
 hw/syn/quartus/vortex/Makefile                |  70 ++
 hw/syn/quartus/vortex/project.tcl             |  41 ++
 .../{top/VX_timing.tcl => vortex/timing.tcl}  |   0
 hw/syn/quartus/vortex/vortex.sdc              |   9 +
 24 files changed, 547 insertions(+), 403 deletions(-)
 create mode 100644 hw/syn/quartus/vortex/Makefile
 create mode 100644 hw/syn/quartus/vortex/project.tcl
 rename hw/syn/quartus/{top/VX_timing.tcl => vortex/timing.tcl} (100%)
 create mode 100644 hw/syn/quartus/vortex/vortex.sdc

diff --git a/driver/rtlsim/Makefile b/driver/rtlsim/Makefile
index f5f32bbc..6627faba 100644
--- a/driver/rtlsim/Makefile
+++ b/driver/rtlsim/Makefile
@@ -35,10 +35,6 @@ VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic $(MULTICORE)
 VL_FLAGS += -Wno-DECLFILENAME
 VL_FLAGS += --x-initial unique
 
-# Use 64 bytes DRAM blocks
-CFLAGS += -DGLOBAL_BLOCK_SIZE=64
-VL_FLAGS += -DGLOBAL_BLOCK_SIZE=64
-
 # Enable Verilator multithreaded simulation
 #THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(max(1, mp.cpu_count() // 2))')
 #VL_FLAGS += --threads $(THREADS)
diff --git a/hw/Makefile b/hw/Makefile
index 21da456a..be559a68 100644
--- a/hw/Makefile
+++ b/hw/Makefile
@@ -8,10 +8,6 @@ VF += --x-initial unique
 
 VF += -exe $(SRCS) $(INCLUDE)
 
-# Use 64 bytes DRAM blocks
-CF += -DGLOBAL_BLOCK_SIZE=64
-VF += -DGLOBAL_BLOCK_SIZE=64
-
 #MULTICORE += -DNUM_CLUSTERS=2 -DNUM_CORES=4
 #MULTICORE += -DNUM_CLUSTERS=1 -DNUM_CORES=4
 MULTICORE += -DNUM_CLUSTERS=1 -DNUM_CORES=2
diff --git a/hw/opae/Makefile b/hw/opae/Makefile
index 4bd86116..a3402664 100644
--- a/hw/opae/Makefile
+++ b/hw/opae/Makefile
@@ -2,7 +2,7 @@
 ASE_BUILD_DIR=build_ase
 FPGA_BUILD_DIR=build_fpga
 
-all: ase ase-1c fpga fpga-1c
+all: ase
 
 ase: setup-ase
 	make -C $(ASE_BUILD_DIR)
diff --git a/hw/opae/README b/hw/opae/README
index efaaf719..ffcf562b 100644
--- a/hw/opae/README
+++ b/hw/opae/README
@@ -51,12 +51,14 @@ make run-fpga
 #
 ## ASE build instructions
 #
-
 source /export/fpga/bin/setup-fpga-env fpga-pac-a10
 
 # Acquire a sever node for running ASE simulations
 qsub-sim
 
+# build
+make ase
+
 # tests
 ./run_ase.sh build_ase ../../driver/tests/basic/basic
 ./run_ase.sh build_ase ../../driver/tests/demo/demo
diff --git a/hw/opae/sources.txt b/hw/opae/sources.txt
index a6039389..83a93aca 100644
--- a/hw/opae/sources.txt
+++ b/hw/opae/sources.txt
@@ -1,9 +1,7 @@
 vortex_afu.json
 
-+define+GLOBAL_BLOCK_SIZE=64
-
-+define+DCACHE_SIZE=2048
-+define+ICACHE_SIZE=1024
++define+DCACHE_SIZE=4096
++define+ICACHE_SIZE=2048
 +define+SCACHE_SIZE=1024
 
 +define+NUM_CORES=2
@@ -11,20 +9,20 @@ vortex_afu.json
 +define+NUM_THREADS=4
 
 +define+DNUM_BANKS=4
-+define+INUM_BANKS=2
++define+INUM_BANKS=1
 +define+SNUM_BANKS=4
 
 +define+DDFPQ_SIZE=16
 +define+IDFPQ_SIZE=16
 +define+SDFPQ_SIZE=0
 
-#+define+DBG_PRINT_CORE_ICACHE
-#+define+DBG_PRINT_CORE_DCACHE
-#+define+DBG_PRINT_CACHE_BANK
-#+define+DBG_PRINT_CACHE_SNP
-#+define+DBG_PRINT_CACHE_MSRQ
-#+define+DBG_PRINT_DRAM
-#+define+DBG_PRINT_OPAE
++define+DBG_PRINT_CORE_ICACHE
++define+DBG_PRINT_CORE_DCACHE
++define+DBG_PRINT_CACHE_BANK
++define+DBG_PRINT_CACHE_SNP
++define+DBG_PRINT_CACHE_MSRQ
++define+DBG_PRINT_DRAM
++define+DBG_PRINT_OPAE
 
 +incdir+.
 +incdir+../rtl
@@ -81,6 +79,7 @@ vortex_afu.json
 ../rtl/libs/VX_generic_priority_encoder.v
 ../rtl/libs/VX_priority_encoder.v
 ../rtl/libs/VX_generic_queue.v
+../rtl/libs/VX_indexable_queue.v
 ../rtl/libs/VX_countones.v
 
 ../rtl/Vortex_Socket.v
diff --git a/hw/opae/vortex_afu.sv b/hw/opae/vortex_afu.sv
index 7c578366..1d0cbdc7 100644
--- a/hw/opae/vortex_afu.sv
+++ b/hw/opae/vortex_afu.sv
@@ -3,7 +3,7 @@ import local_mem_cfg_pkg::*;
 `include "afu_json_info.vh"
 `include "VX_define.vh"
 
-`define DRAM_TO_BYTE_ADDR(x)    {x, 6'b0}
+`define VX_TO_DRAM_ADDR(x)    x[`VX_DRAM_ADDR_WIDTH-1:(`VX_DRAM_ADDR_WIDTH-DRAM_ADDR_WIDTH)] 
 
 module vortex_afu #(
   parameter NUM_LOCAL_MEM_BANKS = 2
@@ -30,17 +30,17 @@ module vortex_afu #(
   output logic [$clog2(NUM_LOCAL_MEM_BANKS)-1:0] mem_bank_select
 );
 
-localparam DRAM_ADDR_WIDTH = $bits(t_local_mem_addr);
-localparam DRAM_LINE_WIDTH = $bits(t_local_mem_data);
-localparam DRAM_TAG_WIDTH  = `L3DRAM_TAG_WIDTH;
+localparam DRAM_ADDR_WIDTH    = $bits(t_local_mem_addr);
+localparam DRAM_LINE_WIDTH    = $bits(t_local_mem_data);
 
-`STATIC_ASSERT(DRAM_ADDR_WIDTH == `L3DRAM_ADDR_WIDTH, "invalid vortex dram bus!")
-`STATIC_ASSERT(DRAM_LINE_WIDTH == `L3DRAM_LINE_WIDTH, "invalid vortex dram bus!")
+localparam DRAM_LINE_LW       = $clog2(DRAM_LINE_WIDTH);
+localparam VX_DRAM_LINE_LW    = $clog2(`VX_DRAM_LINE_WIDTH);
 
 localparam AVS_RD_QUEUE_SIZE  = 16;
 
 localparam CCI_RD_WINDOW_SIZE = 8;
 localparam CCI_RD_QUEUE_SIZE  = 2 * CCI_RD_WINDOW_SIZE;
+localparam CCI_RW_QUEUE_SIZE  = 1024;
 
 localparam AFU_ID_L           = 16'h0002;      // AFU ID Lower
 localparam AFU_ID_H           = 16'h0004;      // AFU ID Higher 
@@ -67,32 +67,33 @@ typedef enum logic[3:0] {
   STATE_CLFLUSH
 } state_t;
 
-typedef logic [`LOG2UP(CCI_RD_WINDOW_SIZE)-1:0] t_cci_rdq_tag;
+typedef logic [$clog2(CCI_RD_WINDOW_SIZE)-1:0] t_cci_rdq_tag;
 typedef logic [$bits(t_ccip_clData) + $bits(t_cci_rdq_tag)-1:0] t_cci_rdq_data;
 
 state_t state;
 
 // Vortex ports ///////////////////////////////////////////////////////////////
 
-logic vx_dram_req_read;
-logic vx_dram_req_write;
-logic [DRAM_ADDR_WIDTH-1:0] vx_dram_req_addr;
-logic [DRAM_LINE_WIDTH-1:0] vx_dram_req_data;
-logic [DRAM_TAG_WIDTH-1:0]  vx_dram_req_tag;
+logic vx_dram_req_valid;
+logic vx_dram_req_rw;
+logic [`VX_DRAM_BYTEEN_WIDTH-1:0] vx_dram_req_byteen;
+logic [`VX_DRAM_ADDR_WIDTH-1:0] vx_dram_req_addr;
+logic [`VX_DRAM_LINE_WIDTH-1:0] vx_dram_req_data;
+logic [`VX_DRAM_TAG_WIDTH-1:0]  vx_dram_req_tag;
 logic vx_dram_req_ready;
 
 logic vx_dram_rsp_valid;
-logic [DRAM_LINE_WIDTH-1:0] vx_dram_rsp_data;
-logic [DRAM_TAG_WIDTH-1:0]  vx_dram_rsp_tag;
+logic [`VX_DRAM_LINE_WIDTH-1:0] vx_dram_rsp_data;
+logic [`VX_DRAM_TAG_WIDTH-1:0]  vx_dram_rsp_tag;
 logic vx_dram_rsp_ready;
 
 logic vx_snp_req_valid;
-logic [DRAM_ADDR_WIDTH-1:0] vx_snp_req_addr;
-logic [0:0] vx_snp_req_tag;
+logic [`VX_DRAM_ADDR_WIDTH-1:0] vx_snp_req_addr;
+logic [`VX_SNP_TAG_WIDTH-1:0] vx_snp_req_tag;
 logic vx_snp_req_ready;
 
 logic vx_snp_rsp_valid;
-logic [0:0] vx_snp_rsp_addr;
+logic [`VX_SNP_TAG_WIDTH-1:0] vx_snp_rsp_tag;
 logic vx_snp_rsp_ready;
 
 logic vx_busy;
@@ -100,14 +101,11 @@ logic vx_busy;
 // AVS Queues /////////////////////////////////////////////////////////////////
 
 logic avs_rtq_push;
-logic [DRAM_TAG_WIDTH-1:0] avs_rtq_din;
 logic avs_rtq_pop;
-logic [DRAM_TAG_WIDTH-1:0] avs_rtq_dout;
 logic avs_rtq_empty;
 logic avs_rtq_full;
 
 logic avs_rdq_push;
-t_local_mem_data avs_rdq_din;
 logic avs_rdq_pop;
 t_local_mem_data avs_rdq_dout;
 logic avs_rdq_empty;
@@ -118,16 +116,11 @@ logic avs_rdq_full;
 logic [2:0]       csr_cmd;
 t_ccip_clAddr     csr_io_addr;
 t_local_mem_addr  csr_mem_addr;
-logic [DRAM_ADDR_WIDTH-1:0] csr_data_size;
+t_ccip_clAddr     csr_data_size;
 
 // MMIO controller ////////////////////////////////////////////////////////////
 
-t_ccip_c0_ReqMmioHdr mmioHdr;
-
-always_comb 
-begin
-  mmioHdr = t_ccip_c0_ReqMmioHdr'(cp2af_sRxPort.c0.hdr);
-end
+t_ccip_c0_ReqMmioHdr mmioHdr = t_ccip_c0_ReqMmioHdr'(cp2af_sRxPort.c0.hdr);
 
 always_ff @(posedge clk) 
 begin
@@ -151,27 +144,27 @@ begin
       case (mmioHdr.address)
         MMIO_CSR_IO_ADDR: begin                     
           csr_io_addr <= t_ccip_clAddr'(cp2af_sRxPort.c0.data);          
-          `ifdef DBG_PRINT_OPAE 
-            $display("%t: CSR_IO_ADDR: 0x%0h", $time, t_ccip_clAddr'(cp2af_sRxPort.c0.data));
-          `endif
+        `ifdef DBG_PRINT_OPAE 
+          $display("%t: CSR_IO_ADDR: 0x%0h", $time, t_ccip_clAddr'(cp2af_sRxPort.c0.data));
+        `endif
         end
         MMIO_CSR_MEM_ADDR: begin          
           csr_mem_addr <= t_local_mem_addr'(cp2af_sRxPort.c0.data);                  
-          `ifdef DBG_PRINT_OPAE
-            $display("%t: CSR_MEM_ADDR: 0x%0h", $time, t_local_mem_addr'(cp2af_sRxPort.c0.data));
-          `endif
+        `ifdef DBG_PRINT_OPAE
+          $display("%t: CSR_MEM_ADDR: 0x%0h", $time, t_local_mem_addr'(cp2af_sRxPort.c0.data));
+        `endif
         end
         MMIO_CSR_DATA_SIZE: begin          
           csr_data_size <= $bits(csr_data_size)'(cp2af_sRxPort.c0.data);          
-          `ifdef DBG_PRINT_OPAE
-            $display("%t: CSR_DATA_SIZE: %0d", $time, $bits(csr_data_size)'(cp2af_sRxPort.c0.data));
-          `endif
+        `ifdef DBG_PRINT_OPAE
+          $display("%t: CSR_DATA_SIZE: %0d", $time, $bits(csr_data_size)'(cp2af_sRxPort.c0.data));
+        `endif
         end
         MMIO_CSR_CMD: begin          
           csr_cmd <= $bits(csr_cmd)'(cp2af_sRxPort.c0.data);
-          `ifdef DBG_PRINT_OPAE
-            $display("%t: CSR_CMD: %0d", $time, $bits(csr_cmd)'(cp2af_sRxPort.c0.data));
-          `endif
+        `ifdef DBG_PRINT_OPAE
+          $display("%t: CSR_CMD: %0d", $time, $bits(csr_cmd)'(cp2af_sRxPort.c0.data));
+        `endif
         end
         default: begin
            // user-defined CSRs
@@ -202,11 +195,11 @@ begin
         16'h0006: af2cp_sTxPort.c2.data <= 64'h0; // next AFU
         16'h0008: af2cp_sTxPort.c2.data <= 64'h0; // reserved
         MMIO_CSR_STATUS: begin
-          `ifdef DBG_PRINT_OPAE
-            if (state != af2cp_sTxPort.c2.data) begin
-              $display("%t: STATUS: state=%0d", $time, state);
-            end
-          `endif
+        `ifdef DBG_PRINT_OPAE
+          if (state != af2cp_sTxPort.c2.data) begin
+            $display("%t: STATUS: state=%0d", $time, state);
+          end
+        `endif
           af2cp_sTxPort.c2.data <= state;
         end  
         default: af2cp_sTxPort.c2.data <= 64'h0;
@@ -218,20 +211,16 @@ end
 
 // COMMAND FSM ////////////////////////////////////////////////////////////////
 
-logic [DRAM_ADDR_WIDTH-1:0] cci_write_ctr;
-logic [DRAM_ADDR_WIDTH-1:0] avs_read_ctr;
-logic [DRAM_ADDR_WIDTH-1:0] avs_write_ctr;
+t_ccip_clAddr               cci_wr_req_ctr;
+logic [DRAM_ADDR_WIDTH-1:0] avs_rd_req_ctr;
+logic [DRAM_ADDR_WIDTH-1:0] avs_wr_req_ctr;
 logic                       vx_reset;
 
 logic cmd_read_done;
 logic cmd_write_done;
-logic cmd_run_done;
 logic cmd_clflush_done;
 
-always_comb 
-begin
-  cmd_run_done = !vx_busy;
-end
+logic cmd_run_done = !vx_busy;
 
 always_ff @(posedge clk) 
 begin
@@ -247,28 +236,28 @@ begin
       STATE_IDLE: begin             
         case (csr_cmd)
           CMD_TYPE_READ: begin     
-            `ifdef DBG_PRINT_OPAE
-              $display("%t: STATE READ: ia=%0h da=%0h sz=%0d", $time, csr_io_addr, csr_mem_addr, csr_data_size);
-            `endif
+          `ifdef DBG_PRINT_OPAE
+            $display("%t: STATE READ: ia=%0h da=%0h sz=%0d", $time, csr_io_addr, csr_mem_addr, csr_data_size);
+          `endif
             state <= STATE_READ;   
           end 
           CMD_TYPE_WRITE: begin      
-            `ifdef DBG_PRINT_OPAE
-              $display("%t: STATE WRITE: ia=%0h da=%0h sz=%0d", $time, csr_io_addr, csr_mem_addr, csr_data_size);
-            `endif
+          `ifdef DBG_PRINT_OPAE
+            $display("%t: STATE WRITE: ia=%0h da=%0h sz=%0d", $time, csr_io_addr, csr_mem_addr, csr_data_size);
+          `endif
             state <= STATE_WRITE;
           end
           CMD_TYPE_RUN: begin        
-            `ifdef DBG_PRINT_OPAE
-              $display("%t: STATE START", $time);
-            `endif
+          `ifdef DBG_PRINT_OPAE
+            $display("%t: STATE START", $time);
+          `endif
             vx_reset <= 1;
             state <= STATE_START;                    
           end
           CMD_TYPE_CLFLUSH: begin
-            `ifdef DBG_PRINT_OPAE
-              $display("%t: STATE CFLUSH: da=%0h sz=%0d", $time, csr_mem_addr, csr_data_size);
-            `endif
+          `ifdef DBG_PRINT_OPAE
+            $display("%t: STATE CFLUSH: da=%0h sz=%0d", $time, csr_mem_addr, csr_data_size);
+          `endif
             state <= STATE_CLFLUSH;
           end
         endcase
@@ -311,116 +300,132 @@ end
 logic vortex_enabled;
 logic cci_rdq_empty;
 t_cci_rdq_data cci_rdq_dout;
-logic cci_rdq_pop;
-logic cci_dram_req_read_fire;
-logic cci_dram_req_write_fire;
-logic vx_dram_req_read_fire;
-logic vx_dram_req_write_fire;
-logic vx_dram_rsp_fire;
-logic [`LOG2UP(AVS_RD_QUEUE_SIZE+1)-1:0] avs_pending_reads, avs_pending_reads_next;
 
-t_ccip_clAddr next_avs_address;
-always_comb 
-begin
-  vortex_enabled = (STATE_RUN == state) || (STATE_CLFLUSH == state);
+logic cci_dram_rd_req_fire;
+logic cci_dram_wr_req_fire;
+logic vx_dram_rd_req_fire;
+logic vx_dram_wr_req_fire;
+logic vx_dram_rd_rsp_fire;
 
-  next_avs_address = csr_mem_addr + {avs_write_ctr[DRAM_ADDR_WIDTH-1:$bits(t_cci_rdq_tag)], t_cci_rdq_tag'(cci_rdq_dout)};
+t_local_mem_byte_mask vx_dram_req_byteen_;
+logic [$clog2(AVS_RD_QUEUE_SIZE+1)-1:0] avs_pending_reads, avs_pending_reads_next;
+logic [DRAM_LINE_LW-1:0] vx_dram_req_offset, vx_dram_rsp_offset;
+logic [DRAM_ADDR_WIDTH-1:0] cci_dram_rd_req_addr, cci_dram_wr_req_addr;
 
-  cci_rdq_pop = (state == STATE_WRITE
-              && !cci_rdq_empty 
-              && !avs_waitrequest
-              && avs_write_ctr < csr_data_size);
+logic cci_dram_rd_req_enable, cci_dram_wr_req_enable;
+logic vx_dram_req_enable, vx_dram_rd_req_enable, vx_dram_wr_req_enable;
 
-  cci_dram_req_read_fire = (state == STATE_READ) 
-                        && (avs_pending_reads < AVS_RD_QUEUE_SIZE)
-                        && !avs_waitrequest 
-                        && avs_read_ctr < csr_data_size;
+assign vortex_enabled = (STATE_RUN == state) || (STATE_CLFLUSH == state);
 
-  cci_dram_req_write_fire = (state == STATE_WRITE) 
-                         && cci_rdq_pop;
+assign cci_dram_rd_req_enable = (state == STATE_READ) 
+                             && (avs_pending_reads < AVS_RD_QUEUE_SIZE)
+                             && (avs_rd_req_ctr != 0);
 
-  vx_dram_req_read_fire  = vx_dram_req_read && vx_dram_req_ready;
+assign cci_dram_wr_req_enable = (state == STATE_WRITE)
+                             && !cci_rdq_empty 
+                             && (avs_wr_req_ctr != 0);
 
-  vx_dram_req_write_fire = vx_dram_req_write && vx_dram_req_ready;
+assign vx_dram_req_enable    = vortex_enabled && (avs_pending_reads < AVS_RD_QUEUE_SIZE);
+assign vx_dram_rd_req_enable = vx_dram_req_enable && vx_dram_req_valid && ~vx_dram_req_rw;
+assign vx_dram_wr_req_enable = vx_dram_req_enable && vx_dram_req_valid && vx_dram_req_rw;
 
-  vx_dram_rsp_fire = vx_dram_rsp_valid && vx_dram_rsp_ready;
+assign cci_dram_rd_req_fire = cci_dram_rd_req_enable && ~avs_waitrequest;
+assign cci_dram_wr_req_fire = cci_dram_wr_req_enable && ~avs_waitrequest;
 
-  if ((cci_dram_req_read_fire || vx_dram_req_read_fire)
-   && ~avs_rdq_pop) begin
-    avs_pending_reads_next = avs_pending_reads + 1;
-  end else 
-  if (~(cci_dram_req_read_fire || vx_dram_req_read_fire)
-   && avs_rdq_pop) begin
-    avs_pending_reads_next = avs_pending_reads - 1;
-  end else begin
-    avs_pending_reads_next = avs_pending_reads;
-  end
+assign vx_dram_rd_req_fire  = vx_dram_rd_req_enable && ~avs_waitrequest;
+assign vx_dram_wr_req_fire  = vx_dram_wr_req_enable && ~avs_waitrequest;
 
-  cmd_write_done = (avs_write_ctr >= csr_data_size);
+assign vx_dram_rd_rsp_fire  = vx_dram_rsp_valid && vx_dram_rsp_ready;
+
+assign avs_pending_reads_next = avs_pending_reads 
+                              + ((cci_dram_rd_req_fire || vx_dram_rd_req_fire) && ~avs_rdq_pop) ? 1 :
+                                (~(cci_dram_rd_req_fire || vx_dram_rd_req_fire) && avs_rdq_pop) ? -1 : 0;
+
+assign cmd_write_done = (0 == avs_wr_req_ctr);
+
+if (`VX_DRAM_LINE_WIDTH != DRAM_LINE_WIDTH) begin
+  assign vx_dram_req_offset  = {{VX_DRAM_LINE_LW{1'b0}}, vx_dram_req_addr[(DRAM_LINE_LW-VX_DRAM_LINE_LW)-1:0]} << VX_DRAM_LINE_LW;    
+  assign vx_dram_req_byteen_ = vx_dram_req_byteen << ({(VX_DRAM_LINE_LW - 3)'(0), vx_dram_req_addr[(DRAM_LINE_LW-VX_DRAM_LINE_LW)-1:0]} << (VX_DRAM_LINE_LW - 3));
+end else begin
+  assign vx_dram_req_offset  = 0;
+  assign vx_dram_req_byteen_ = 64'hffffffffffffffff;
 end
 
+always_comb 
+begin        
+  case (state)
+    CMD_TYPE_READ:  avs_address = cci_dram_rd_req_addr;
+    CMD_TYPE_WRITE: avs_address = cci_dram_wr_req_addr;
+    default:        avs_address = `VX_TO_DRAM_ADDR(vx_dram_req_addr);
+  endcase
+
+  case (state)
+    CMD_TYPE_READ:  avs_byteenable = 64'hffffffffffffffff;
+    CMD_TYPE_WRITE: avs_byteenable = 64'hffffffffffffffff;
+    default:        avs_byteenable = vx_dram_req_byteen_;
+  endcase
+
+  case (state)
+    CMD_TYPE_WRITE: avs_writedata = cci_rdq_dout[$bits(t_ccip_clData) + $bits(t_cci_rdq_tag)-1:$bits(t_cci_rdq_tag)];
+    default:        avs_writedata = vx_dram_req_data << vx_dram_req_offset;
+  endcase
+end
+
+assign avs_read  = cci_dram_rd_req_enable || vx_dram_rd_req_enable;
+assign avs_write = cci_dram_wr_req_enable || vx_dram_wr_req_enable;
+
 always_ff @(posedge clk) 
 begin
   if (SoftReset) 
   begin    
-    mem_bank_select   <= 0;
-    avs_burstcount    <= 1;
-    avs_byteenable    <= 64'hffffffffffffffff; 
-    avs_read          <= 0;
-    avs_write         <= 0;
-    avs_read_ctr      <= 0;
-    avs_write_ctr     <= 0;
-    avs_pending_reads <= 0;
+    mem_bank_select      <= 0;
+    avs_burstcount       <= 1;
+    avs_rd_req_ctr       <= 0;
+    avs_wr_req_ctr       <= 0;
+    avs_pending_reads    <= 0;
+    cci_dram_rd_req_addr <= 0;
+    cci_dram_wr_req_addr <= 0;
   end
   else begin
-
-    avs_read <= 0;
-    avs_write <= 0;
-
-    if (state == STATE_IDLE) begin
-      avs_read_ctr  <= 0;
-      avs_write_ctr <= 0;
-    end
-
-    if (cci_dram_req_read_fire) begin
-      avs_address  <= csr_mem_addr + avs_read_ctr;          
-      avs_read_ctr <= avs_read_ctr + 1;          
-      avs_read     <= 1;
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: AVS Rd Req: addr=%0h, pending=%0d", $time, `DRAM_TO_BYTE_ADDR(csr_mem_addr + avs_read_ctr), avs_pending_reads);
-      `endif
-    end
-
-    if (cci_dram_req_write_fire) begin          
-      avs_writedata <= cci_rdq_dout[$bits(t_ccip_clData) + $bits(t_cci_rdq_tag)-1:$bits(t_cci_rdq_tag)];
-      avs_address   <= next_avs_address;          
-      avs_write_ctr <= avs_write_ctr + 1;
-      avs_write     <= 1;
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: AVS Wr Req: addr=%0h (%0d/%0d)", $time, `DRAM_TO_BYTE_ADDR(next_avs_address), avs_write_ctr + 1, csr_data_size);
-      `endif
-    end
-
-    if (vx_dram_req_read_fire) begin
-      avs_address <= vx_dram_req_addr;
-      avs_read    <= 1;
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: AVS Rd Req: addr=%0h, pending=%0d", $time, `DRAM_TO_BYTE_ADDR(vx_dram_req_addr), avs_pending_reads);
-      `endif
-    end 
     
-    if (vx_dram_req_write_fire) begin
-      avs_address   <= vx_dram_req_addr;
-      avs_writedata <= vx_dram_req_data;          
-      avs_write     <= 1;
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: AVS Wr Req: addr=%0h", $time, `DRAM_TO_BYTE_ADDR(vx_dram_req_addr));
-      `endif
-    end   
+    if (state == STATE_IDLE) begin
+      if (CMD_TYPE_READ == csr_cmd) begin
+        cci_dram_rd_req_addr <= csr_mem_addr;
+        avs_rd_req_ctr       <= csr_data_size;
+      end 
+      else if (CMD_TYPE_WRITE == csr_cmd) begin
+        cci_dram_wr_req_addr <= csr_mem_addr;
+        avs_wr_req_ctr       <= csr_data_size;
+      end
+    end
+
+    if (cci_dram_rd_req_fire) begin
+      cci_dram_rd_req_addr <= cci_dram_rd_req_addr + 1;       
+      avs_rd_req_ctr       <= avs_rd_req_ctr - 1;  
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: AVS Rd Req: addr=%0h, rem=%0d, pending=%0d", $time, `DRAM_TO_BYTE_ADDR(avs_address), (avs_rd_req_ctr - 1), avs_pending_reads_next);
+    `endif
+    end
+
+    if (cci_dram_wr_req_fire) begin                
+      cci_dram_wr_req_addr <= ((cci_dram_wr_req_addr + 1) & ~(CCI_RD_WINDOW_SIZE-1)) | t_cci_rdq_tag'(cci_rdq_dout);
+      avs_wr_req_ctr       <= avs_wr_req_ctr - 1;
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: AVS Wr Req: addr=%0h, data=%0h, rem=%0d", $time, `DRAM_TO_BYTE_ADDR(avs_address), avs_writedata, (avs_wr_req_ctr - 1));
+    `endif
+    end    
 
     `ifdef DBG_PRINT_OPAE
+      if (vx_dram_rd_req_fire) begin
+        $display("%t: AVS Rd Req: addr=%0h, byteen=%0h, tag=%0h, pending=%0d", $time, `DRAM_TO_BYTE_ADDR(avs_address), avs_byteenable, vx_dram_req_tag, avs_pending_reads_next);
+      end 
+      
+      if (vx_dram_wr_req_fire) begin
+        $display("%t: AVS Wr Req: addr=%0h, byteen=%0h, tag=%0h, data=%0h", $time, `DRAM_TO_BYTE_ADDR(avs_address), avs_byteenable, vx_dram_req_tag, avs_writedata);
+      end   
+
       if (avs_readdatavalid) begin
-        $display("%t: AVS Rd Rsp: pending=%0d", $time, avs_pending_reads_next);
+        $display("%t: AVS Rd Rsp: data=%0h, pending=%0d", $time, avs_readdata, avs_pending_reads_next);
       end
     `endif
 
@@ -430,55 +435,42 @@ end
 
 // Vortex DRAM requests
 
-always_comb 
-begin  
-  vx_dram_req_ready = vortex_enabled 
-                   && !avs_waitrequest 
-                   && (avs_pending_reads < AVS_RD_QUEUE_SIZE);
-end
+assign vx_dram_req_ready = vx_dram_req_enable && !avs_waitrequest;
 
 // Vortex DRAM fill response
 
-always_comb 
-begin
-  vx_dram_rsp_valid = vortex_enabled && !avs_rdq_empty;
-  vx_dram_rsp_tag   = avs_rtq_dout;
-  vx_dram_rsp_data  = avs_rdq_dout;
+assign vx_dram_rsp_valid = vortex_enabled && !avs_rdq_empty;
+if (`VX_DRAM_LINE_WIDTH != DRAM_LINE_WIDTH) begin
+  assign vx_dram_rsp_data = (avs_rdq_dout >> vx_dram_rsp_offset);
+end else begin
+  assign vx_dram_rsp_data = avs_rdq_dout;    
 end
 
 // AVS address read request queue /////////////////////////////////////////////
 
-logic cci_wr_req;
-
-always_comb 
-begin
-  avs_rtq_push = vx_dram_req_read_fire;
-  avs_rtq_din  = vx_dram_req_tag;
-  avs_rtq_pop  = vx_dram_rsp_fire;
-end
+assign avs_rtq_push = vx_dram_rd_req_fire;
+assign avs_rtq_pop  = vx_dram_rd_rsp_fire;
 
 VX_generic_queue #(
-  .DATAW(DRAM_TAG_WIDTH),
+  .DATAW(`VX_DRAM_TAG_WIDTH + DRAM_LINE_LW),
   .SIZE(AVS_RD_QUEUE_SIZE)
 ) avs_rd_req_queue (
   .clk      (clk),
   .reset    (SoftReset),
   .push     (avs_rtq_push),
-  .data_in  (avs_rtq_din),
+  .data_in  ({vx_dram_req_tag, vx_dram_req_offset}),
   .pop      (avs_rtq_pop),
-  .data_out (avs_rtq_dout),
+  .data_out ({vx_dram_rsp_tag, vx_dram_rsp_offset}),
   .empty    (avs_rtq_empty),
   .full     (avs_rtq_full)
 );
 
 // AVS data read response queue ///////////////////////////////////////////////
 
-always_comb 
-begin
-  avs_rdq_push = avs_readdatavalid;
-  avs_rdq_din  = avs_readdata;
-  avs_rdq_pop  = vx_dram_rsp_fire || cci_wr_req; 
-end
+logic cci_wr_req_fire;
+
+assign avs_rdq_push = avs_readdatavalid;
+assign avs_rdq_pop  = vx_dram_rd_rsp_fire || cci_wr_req_fire; 
 
 VX_generic_queue #(
   .DATAW(DRAM_LINE_WIDTH),
@@ -487,81 +479,102 @@ VX_generic_queue #(
   .clk      (clk),
   .reset    (SoftReset),
   .push     (avs_rdq_push),
-  .data_in  (avs_rdq_din),
+  .data_in  (avs_readdata),
   .pop      (avs_rdq_pop),
   .data_out (avs_rdq_dout),
   .empty    (avs_rdq_empty),
   .full     (avs_rdq_full)
 );
 
-// CCI Read Request ///////////////////////////////////////////////////////////
+// CCI-P Read Request ///////////////////////////////////////////////////////////
 
-t_ccip_c0_ReqMemHdr cci_read_hdr;
+logic [$clog2(CCI_RD_QUEUE_SIZE+1)-1:0] cci_pending_reads, cci_pending_reads_next;
+t_ccip_clAddr cci_rd_req_addr, cci_rd_req_ctr, cci_rd_req_ctr_next;
+t_cci_rdq_tag cci_rd_rsp_ctr;
 
-logic [DRAM_ADDR_WIDTH-1:0] cci_read_ctr;
-t_cci_rdq_tag cci_rdq_ctr;
+logic cci_rd_req_fire, cci_rd_rsp_fire;
+logic cci_rd_req_enable, cci_rd_req_wait;
 
-logic cci_rdq_full;
-logic cci_rdq_push;
+logic cci_rdq_full, cci_rdq_push, cci_rdq_pop;
 t_cci_rdq_data cci_rdq_din;
 
-logic cci_read_wait;
-
-always_comb 
-begin
-  cci_read_hdr = t_ccip_c0_ReqMemHdr'(0);
-  cci_read_hdr.address = csr_io_addr + cci_read_ctr;  
-  cci_read_hdr.mdata = t_cci_rdq_tag'(cci_read_ctr);
-
-  cci_rdq_push = (STATE_WRITE == state) && cp2af_sRxPort.c0.rspValid;
-  cci_rdq_din  = {cp2af_sRxPort.c0.data, t_cci_rdq_tag'(cp2af_sRxPort.c0.hdr.mdata)};  
+always_comb begin
+  af2cp_sTxPort.c0.hdr         = t_ccip_c0_ReqMemHdr'(0);
+  af2cp_sTxPort.c0.hdr.address = cci_rd_req_addr;  
+  af2cp_sTxPort.c0.hdr.mdata   = t_cci_rdq_tag'(cci_rd_req_ctr);
 end
 
+assign cci_rd_req_fire = af2cp_sTxPort.c0.valid && !cp2af_sRxPort.c0TxAlmFull;
+assign cci_rd_rsp_fire = (STATE_WRITE == state) && cp2af_sRxPort.c0.rspValid;
+
+assign cci_rd_req_ctr_next = cci_rd_req_ctr + (cci_rd_req_fire ? 1 : 0);
+
+assign cci_rdq_pop  = cci_dram_wr_req_fire;
+assign cci_rdq_push = cci_rd_rsp_fire;
+assign cci_rdq_din  = {cp2af_sRxPort.c0.data, t_cci_rdq_tag'(cp2af_sRxPort.c0.hdr.mdata)};  
+
+assign cci_pending_reads_next = cci_pending_reads 
+                              + (cci_rd_req_fire && ~cci_rdq_pop) ? 1 : 
+                                (~cci_rd_req_fire && cci_rdq_pop) ? -1 : 0;
+
+assign af2cp_sTxPort.c0.valid = cci_rd_req_enable && ~cci_rd_req_wait;
+
 // Send read requests to CCI
 always_ff @(posedge clk) 
 begin
   if (SoftReset) begin
-    af2cp_sTxPort.c0.hdr   <= 0;
-    af2cp_sTxPort.c0.valid <= 0;
-    cci_read_ctr           <= 0;
-    cci_rdq_ctr            <= 0;
-    cci_read_wait          <= 0;
+    cci_rd_req_addr   <= 0;
+    cci_rd_req_ctr    <= 0;
+    cci_rd_rsp_ctr    <= 0;
+    cci_pending_reads <= 0;
+    cci_rd_req_enable <= 0;
+    cci_rd_req_wait   <= 0;
   end 
   else begin      
-    af2cp_sTxPort.c0.valid <= 0;
-
-    if (STATE_IDLE == state) begin
-      cci_read_ctr  <= 0;
-      cci_rdq_ctr   <= 0;
-      cci_read_wait <= 0;
+    
+    if ((STATE_IDLE == state) 
+    &&  (CMD_TYPE_WRITE == csr_cmd)) begin
+      cci_rd_req_addr   <= csr_io_addr;
+      cci_rd_req_ctr    <= 0;
+      cci_rd_rsp_ctr    <= 0;
+      cci_pending_reads <= 0;
+      cci_rd_req_enable <= (csr_data_size != 0);
+      cci_rd_req_wait   <= 0;
     end
 
-    if (STATE_WRITE == state 
-     && !cp2af_sRxPort.c0TxAlmFull      // ensure read queue not full
-     && !cci_rdq_full                   // ensure destination queue not full
-     && !cci_read_wait                  // ensure the last batch has arrived
-     && cci_read_ctr < csr_data_size)   // ensure not done
-    begin
-      af2cp_sTxPort.c0.hdr   <= cci_read_hdr;
-      af2cp_sTxPort.c0.valid <= 1;      
-      cci_read_ctr           <= cci_read_ctr + 1;
-      if (t_cci_rdq_tag'(cci_read_ctr) == (CCI_RD_WINDOW_SIZE-1)) begin
-        cci_read_wait <= 1;             // end current request batch
+    cci_rd_req_enable <= (STATE_WRITE == state)                       
+                      && (cci_rd_req_ctr_next < csr_data_size)
+                      && (cci_pending_reads_next < CCI_RD_QUEUE_SIZE);    
+
+    if (cci_rd_req_fire) begin  
+      cci_rd_req_addr <= cci_rd_req_addr + 1;
+      cci_rd_req_ctr  <= cci_rd_req_ctr_next;
+      if (t_cci_rdq_tag'(cci_rd_req_ctr) == (CCI_RD_WINDOW_SIZE-1)) begin
+        cci_rd_req_wait <= 1;   // end current request batch
       end 
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: CCI Rd Req: addr=%0h, ctr=%0d", $time, `DRAM_TO_BYTE_ADDR(cci_read_hdr.address), cci_read_ctr);
-      `endif
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: CCI Rd Req: addr=%0h, rem=%0d, pending=%0d", $time, cci_rd_req_addr, (csr_data_size - cci_rd_req_ctr_next), cci_pending_reads_next);
+    `endif
     end
 
-    if (cci_rdq_push) begin
-      cci_rdq_ctr <= cci_rdq_ctr + 1;
-      if (cci_rdq_ctr == (CCI_RD_WINDOW_SIZE-1)) begin
-        cci_read_wait <= 0;             // restart new request batch
+    if (cci_rd_rsp_fire) begin
+      cci_rd_rsp_ctr <= cci_rd_rsp_ctr + 1;
+      if (cci_rd_rsp_ctr == (CCI_RD_WINDOW_SIZE-1)) begin
+        cci_rd_req_wait <= 0;   // restart new request batch
       end 
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: CCI Rd Rsp: idx=%0d, ctr=%0d", $time, t_cci_rdq_tag'(cp2af_sRxPort.c0.hdr.mdata), cci_rdq_ctr);
-      `endif
-    end        
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: CCI Rd Rsp: idx=%0d, ctr=%0d", $time, t_cci_rdq_tag'(cp2af_sRxPort.c0.hdr.mdata), cci_rd_rsp_ctr);
+    `endif
+    end
+
+    if (cci_rdq_pop) begin
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: CCI Rd Queue Pop: pending=%0d", $time, cci_pending_reads_next);
+    `endif
+    end
+
+    cci_pending_reads <= cci_pending_reads_next;
+
   end
 end
 
@@ -579,67 +592,65 @@ VX_generic_queue #(
   .full     (cci_rdq_full)
 );
 
-// CCI Write Request //////////////////////////////////////////////////////////
+// CCI-P Write Request //////////////////////////////////////////////////////////
 
-t_ccip_c1_ReqMemHdr cci_write_hdr;
+logic [$clog2(CCI_RW_QUEUE_SIZE+1)-1:0] cci_pending_writes, cci_pending_writes_next;
+t_ccip_clAddr cci_wr_req_addr;
+logic cci_wr_req_enable, cci_wr_rsp_fire;
 
-logic [DRAM_ADDR_WIDTH:0] cci_pending_writes, cci_pending_writes_next;
+always_comb begin
+  af2cp_sTxPort.c1.hdr         = t_ccip_c1_ReqMemHdr'(0);
+  af2cp_sTxPort.c1.hdr.address = cci_wr_req_addr;
+  af2cp_sTxPort.c1.hdr.sop     = 1; // single line write mode
+  af2cp_sTxPort.c1.data        = t_ccip_clData'(avs_rdq_dout);  
+end 
 
-always_comb 
-begin
-  cci_wr_req = (STATE_READ == state) 
-            && !avs_rdq_empty 
-            && !cp2af_sRxPort.c1TxAlmFull
-            && (cci_write_ctr < csr_data_size);  
+assign cci_wr_req_fire = af2cp_sTxPort.c1.valid && !cp2af_sRxPort.c1TxAlmFull;
+assign cci_wr_rsp_fire = (STATE_READ == state) && cp2af_sRxPort.c1.rspValid;
 
-  if (cci_wr_req && ~cp2af_sRxPort.c1.rspValid) begin
-    cci_pending_writes_next = cci_pending_writes + 1;
-  end else
-  if (~cci_wr_req && cp2af_sRxPort.c1.rspValid) begin
-    cci_pending_writes_next = cci_pending_writes - 1;
-  end else begin
-    cci_pending_writes_next = cci_pending_writes;
-  end
+assign cci_pending_writes_next = cci_pending_writes 
+                               + (cci_wr_req_fire && ~cci_wr_rsp_fire) ? 1 :
+                                 (~cci_wr_req_fire && cci_wr_rsp_fire) ? -1 : 0;
 
-  cci_write_hdr = t_ccip_c1_ReqMemHdr'(0);
-  cci_write_hdr.address = csr_io_addr + cci_write_ctr;
-  cci_write_hdr.sop = 1; // single line write mode
+assign cmd_read_done = (0 == cci_wr_req_ctr) && (0 == cci_pending_writes);
 
-  cmd_read_done = (cci_write_ctr >= csr_data_size) && (0 == cci_pending_writes);
-end
+assign af2cp_sTxPort.c1.valid = cci_wr_req_enable && ~avs_rdq_empty;
 
 // Send write requests to CCI
 always_ff @(posedge clk) 
 begin
   if (SoftReset) begin
-    af2cp_sTxPort.c1.hdr   <= 0;
-    af2cp_sTxPort.c1.data  <= 0;
-    af2cp_sTxPort.c1.valid <= 0;
-    cci_write_ctr          <= 0;
-    cci_pending_writes     <= 0;
+    cci_wr_req_addr    <= 0;
+    cci_wr_req_ctr     <= 0;
+    cci_wr_req_enable  <= 0;
+    cci_pending_writes <= 0;
   end
   else begin
-    af2cp_sTxPort.c1.valid <= 0;
+    
+    if ((STATE_IDLE == state) 
+    &&  (CMD_TYPE_READ == csr_cmd)) begin
+      cci_wr_req_addr    <= csr_io_addr;
+      cci_wr_req_ctr     <= csr_data_size;
+      cci_pending_writes <= 0;
+    end    
 
-    if (STATE_IDLE == state) begin
-      cci_write_ctr <= 0;
-    end
-
-    if (cci_wr_req) begin
-      af2cp_sTxPort.c1.hdr   <= cci_write_hdr;
-      af2cp_sTxPort.c1.data  <= t_ccip_clData'(avs_rdq_dout);
-      af2cp_sTxPort.c1.valid <= 1;            
-      cci_write_ctr          <= cci_write_ctr + 1;
-      `ifdef DBG_PRINT_OPAE
-        $display("%t: CCI Wr Req: addr=%0h (%0d/%0d)", $time, `DRAM_TO_BYTE_ADDR(cci_write_hdr.address), cci_write_ctr + 1, csr_data_size);
-      `endif
-    end
+    cci_wr_req_enable <= (STATE_READ == state) 
+                      && (cci_pending_writes_next < CCI_RW_QUEUE_SIZE);    
 
+    if (cci_wr_req_fire) begin
+      assert(cci_wr_req_ctr != 0);  
+      cci_wr_req_addr <= cci_wr_req_addr + 1;        
+      cci_wr_req_ctr  <= cci_wr_req_ctr - 1;
     `ifdef DBG_PRINT_OPAE
-      if (cp2af_sRxPort.c1.rspValid) begin      
-        $display("%t: CCI Wr Rsp: pending=%0d", $time, cci_pending_writes_next);      
-      end
+      $display("%t: CCI Wr Req: addr=%0h, rem=%0d, pending=%0d", $time, cci_wr_req_addr, (cci_wr_req_ctr - 1), cci_pending_writes_next);
     `endif
+    end
+
+  `ifdef DBG_PRINT_OPAE
+    if (cci_wr_rsp_fire) begin      
+      $display("%t: CCI Wr Rsp: pending=%0d", $time, cci_pending_writes_next);      
+    end
+  `endif
 
     cci_pending_writes <= cci_pending_writes_next;
   end
@@ -647,49 +658,72 @@ end
 
 // Vortex cache snooping //////////////////////////////////////////////////////
 
-logic [DRAM_ADDR_WIDTH-1:0] snp_req_ctr;
-logic [DRAM_ADDR_WIDTH-1:0] snp_rsp_ctr;
+logic [`VX_DRAM_ADDR_WIDTH-1:0] snp_req_size;
+logic [`VX_DRAM_ADDR_WIDTH-1:0] snp_req_baseaddr;
+logic [`VX_DRAM_ADDR_WIDTH-1:0] snp_req_ctr, snp_rsp_ctr;
 
-logic vx_snp_rsp_fire;
+logic vx_snp_req_fire, vx_snp_rsp_fire;
 
-always_comb 
-begin
-  cmd_clflush_done = (snp_rsp_ctr >= csr_data_size);
-  vx_snp_rsp_fire = vx_snp_rsp_valid && vx_snp_rsp_ready;
+if (`VX_DRAM_LINE_WIDTH != DRAM_LINE_WIDTH) begin
+  assign snp_req_baseaddr = {csr_mem_addr, (`VX_DRAM_ADDR_WIDTH - DRAM_ADDR_WIDTH)'(0)};
+  assign snp_req_size     = {csr_data_size, (`VX_DRAM_ADDR_WIDTH - DRAM_ADDR_WIDTH)'(0)};
+end else begin
+  assign snp_req_baseaddr = csr_mem_addr;
+  assign snp_req_size     = csr_data_size;
 end
 
+assign vx_snp_req_fire  = vx_snp_req_valid && vx_snp_req_ready;
+assign vx_snp_rsp_fire  = vx_snp_rsp_valid && vx_snp_rsp_ready;
+assign cmd_clflush_done = (0 == snp_rsp_ctr);  
+
 always_ff @(posedge clk) 
 begin
   if (SoftReset) begin
     vx_snp_req_valid <= 0;
+    vx_snp_req_addr  <= 0;
     vx_snp_req_tag   <= 0;
     vx_snp_rsp_ready <= 0;
     snp_req_ctr      <= 0;
     snp_rsp_ctr      <= 0;
   end
   else begin
-    if (STATE_IDLE == state) begin
-      snp_req_ctr      <= 0;
-      snp_rsp_ctr      <= 0;
-      vx_snp_rsp_ready <= 0;
-    end
 
-    vx_snp_req_valid <= 0;
-
-    if ((STATE_CLFLUSH == state)
-     && (snp_req_ctr < csr_data_size)
-     && vx_snp_req_ready)
-    begin
-      vx_snp_req_addr  <= csr_mem_addr + snp_req_ctr;
-      snp_req_ctr      <= snp_req_ctr + 1;
-      vx_snp_req_valid <= 1;
-      vx_snp_rsp_ready <= 1;
+    if ((STATE_IDLE == state) 
+    &&  (CMD_TYPE_CLFLUSH == csr_cmd)) begin
+      vx_snp_req_addr  <= snp_req_baseaddr;
+      snp_req_ctr      <= snp_req_size;
+      snp_rsp_ctr      <= snp_req_size;
+      vx_snp_req_valid <= (snp_req_size != 0);
+      vx_snp_rsp_ready <= (snp_req_size != 0);
+    end
+
+    if ((STATE_CLFLUSH == state) 
+     && (0 == snp_rsp_ctr)) begin
+       vx_snp_rsp_ready <= 0;
+    end
+
+    if ((STATE_CLFLUSH == state) 
+     && (0 == snp_req_ctr)) begin
+       vx_snp_req_valid <= 0;
+    end
+
+    if (vx_snp_req_fire)
+    begin
+      vx_snp_req_addr <= vx_snp_req_addr + 1;
+      vx_snp_req_tag  <= snp_req_ctr[`VX_SNP_TAG_WIDTH-1:0];
+      snp_req_ctr     <= snp_req_ctr - 1;
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: AFU Snp Req: addr=%0h, tag=%0d, rem=%0d", $time, `DRAM_TO_BYTE_ADDR(vx_snp_req_addr), vx_snp_req_tag, (snp_req_ctr - 1));
+    `endif
     end
 
     if ((STATE_CLFLUSH == state) 
-     && (snp_rsp_ctr < csr_data_size)
      && vx_snp_rsp_fire) begin
-       snp_rsp_ctr <= snp_rsp_ctr + 1;
+       assert(snp_rsp_ctr != 0);
+       snp_rsp_ctr <= snp_rsp_ctr - 1;
+    `ifdef DBG_PRINT_OPAE
+      $display("%t: AFU Snp Rsp: tag=%0d, rem=%0d", $time, vx_snp_rsp_tag, (snp_rsp_ctr - 1));
+    `endif
     end   
   end
 end
@@ -701,8 +735,9 @@ Vortex_Socket #() vx_socket (
   .reset            (vx_reset),
 
   // DRAM request 
-  .dram_req_write   (vx_dram_req_write),
-  .dram_req_read 	  (vx_dram_req_read),
+  .dram_req_valid   (vx_dram_req_valid),
+  .dram_req_rw   	  (vx_dram_req_rw),
+  .dram_req_byteen  (vx_dram_req_byteen),
   .dram_req_addr 		(vx_dram_req_addr),
   .dram_req_data		(vx_dram_req_data),
   .dram_req_tag     (vx_dram_req_tag),
@@ -726,18 +761,18 @@ Vortex_Socket #() vx_socket (
   .snp_rsp_ready    (vx_snp_rsp_ready),
 
   // I/O request
-  .io_req_read      (),
-  .io_req_write     (),    
+  .io_req_valid     (),
+  .io_req_rw        (),
+  .io_req_byteen    (),    
   .io_req_addr      (),
-  .io_req_data      (),
-  .io_req_byteen    (),
+  .io_req_data      (),  
   .io_req_tag       (),    
-  .io_req_ready     (1'b1),
+  .io_req_ready     (1),
 
   // I/O response
-  .io_rsp_valid     (1'b0),
-  .io_rsp_data      (32'b0),
-  .io_rsp_tag       (`DCORE_TAG_WIDTH'(0)),
+  .io_rsp_valid     (0),
+  .io_rsp_data      (0),
+  .io_rsp_tag       (0),
   .io_rsp_ready     (),
  
   // status
diff --git a/hw/rtl/VX_config.vh b/hw/rtl/VX_config.vh
index d7085bf5..bee56a25 100644
--- a/hw/rtl/VX_config.vh
+++ b/hw/rtl/VX_config.vh
@@ -77,7 +77,7 @@
 
 // Size of cache in bytes
 `ifndef DCACHE_SIZE
-`define DCACHE_SIZE 2048
+`define DCACHE_SIZE 4096
 `endif
 
 // Size of line inside a bank in bytes
diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh
index d6dab389..cd4c5029 100644
--- a/hw/rtl/VX_define.vh
+++ b/hw/rtl/VX_define.vh
@@ -136,8 +136,7 @@
 
 ///////////////////////////////////////////////////////////////////////////////
 
-`ifndef NDEBUG
-                                  // pc,  wb, rd, warp_num
+`ifndef NDEBUG                    // pc,  wb, rd, warp_num
 `define DEBUG_CORE_REQ_MDATA_WIDTH  (32 + 2 + 5 + `NW_BITS)
 `else
 `define DEBUG_CORE_REQ_MDATA_WIDTH  0
@@ -227,10 +226,10 @@
 `define L2CACHE_ID          (`L3_ENABLE ? 1 : 0)
 
 // DRAM request data bits
-`define L2DRAM_LINE_WIDTH   (`L2BANK_LINE_SIZE * 8)
+`define L2DRAM_LINE_WIDTH   (`L2_ENABLE ? (`L2BANK_LINE_SIZE * 8) : `DDRAM_LINE_WIDTH)
 
 // DRAM request address bits
-`define L2DRAM_ADDR_WIDTH   (32 - `CLOG2(`L2BANK_LINE_SIZE))
+`define L2DRAM_ADDR_WIDTH   (`L2_ENABLE ? (32 - `CLOG2(`L2BANK_LINE_SIZE)) : `DDRAM_ADDR_WIDTH)
 
 // DRAM byte enable bits
 `define L2DRAM_BYTEEN_WIDTH (`L2_ENABLE ? `L2BANK_LINE_SIZE : `DDRAM_BYTEEN_WIDTH)
@@ -242,7 +241,7 @@
 `define L2SNP_TAG_WIDTH     (`L3_ENABLE ? `LOG2UP(`L3SNRQ_SIZE) : `L3SNP_TAG_WIDTH)
 
 // Number of Word requests per cycle {1, 2, 4, 8, ...}
-`define L2NUM_REQUESTS      (2*`NUM_CORES)
+`define L2NUM_REQUESTS      (2 * `NUM_CORES)
 
 ////////////////////////// L3cache Configurable Knobs /////////////////////////
 
@@ -250,10 +249,10 @@
 `define L3CACHE_ID          0
 
 // DRAM request data bits
-`define L3DRAM_LINE_WIDTH   (`L3BANK_LINE_SIZE * 8)
+`define L3DRAM_LINE_WIDTH   (`L3_ENABLE ? (`L3BANK_LINE_SIZE * 8) : `L2DRAM_LINE_WIDTH)
 
 // DRAM request address bits
-`define L3DRAM_ADDR_WIDTH   (32 - `CLOG2(`L3BANK_LINE_SIZE))
+`define L3DRAM_ADDR_WIDTH   (`L3_ENABLE ? (32 - `CLOG2(`L3BANK_LINE_SIZE)) : `L2DRAM_ADDR_WIDTH)
 
 // DRAM byte enable bits
 `define L3DRAM_BYTEEN_WIDTH (`L3_ENABLE ? `L3BANK_LINE_SIZE : `L2DRAM_BYTEEN_WIDTH)
@@ -267,5 +266,16 @@
 // Number of Word requests per cycle {1, 2, 4, 8, ...}
 `define L3NUM_REQUESTS      `NUM_CLUSTERS
 
+///////////////////////////////////////////////////////////////////////////////
+
+`define VX_DRAM_BYTEEN_WIDTH    `L3DRAM_BYTEEN_WIDTH   
+`define VX_DRAM_ADDR_WIDTH      `L3DRAM_ADDR_WIDTH
+`define VX_DRAM_LINE_WIDTH      `L3DRAM_LINE_WIDTH
+`define VX_DRAM_TAG_WIDTH       `L3DRAM_TAG_WIDTH
+`define VX_SNP_TAG_WIDTH        `L3SNP_TAG_WIDTH    
+`define VX_CORE_TAG_WIDTH       `DCORE_TAG_WIDTH 
+
+`define DRAM_TO_BYTE_ADDR(x)     {x, (32-$bits(x))'(0)}
+
  // VX_DEFINE
 `endif
diff --git a/hw/rtl/VX_dmem_ctrl.v b/hw/rtl/VX_dmem_ctrl.v
index 523f44a8..b7ecf5eb 100644
--- a/hw/rtl/VX_dmem_ctrl.v
+++ b/hw/rtl/VX_dmem_ctrl.v
@@ -39,7 +39,7 @@ module VX_dmem_ctrl # (
     ) dcache_core_rsp_qual_if(), smem_core_rsp_if();
 
     // use "case equality" to handle uninitialized entry
-    wire smem_select = ((dcache_core_req_if.core_req_addr[0] >= `BYTE_TO_WORD_ADDR(`SHARED_MEM_BASE_ADDR, `DWORD_SIZE)) === 1'b1);
+    wire smem_select = (({dcache_core_req_if.core_req_addr[0], 2'b0} >= `SHARED_MEM_BASE_ADDR) === 1'b1);
 
     VX_dcache_io_arb dcache_io_arb (
         .io_select          (smem_select),
diff --git a/hw/rtl/VX_icache_stage.v b/hw/rtl/VX_icache_stage.v
index e8e04525..79330f5c 100644
--- a/hw/rtl/VX_icache_stage.v
+++ b/hw/rtl/VX_icache_stage.v
@@ -64,7 +64,7 @@ module VX_icache_stage #(
     // Icache Request
     assign icache_req_if.core_req_valid  = valid_inst && ~mrq_full;
     assign icache_req_if.core_req_rw     = 0;
-    assign icache_req_if.core_req_byteen = 0;
+    assign icache_req_if.core_req_byteen = 4'b1111;
     assign icache_req_if.core_req_addr   = fe_inst_meta_fi.inst_pc[31:2];
     assign icache_req_if.core_req_data   = 0;    
 
diff --git a/hw/rtl/VX_lsu_unit.v b/hw/rtl/VX_lsu_unit.v
index 91bccbe6..8b460b96 100644
--- a/hw/rtl/VX_lsu_unit.v
+++ b/hw/rtl/VX_lsu_unit.v
@@ -36,6 +36,8 @@ module VX_lsu_unit #(
     wire[1:0]                       use_wb;
     wire[31:0]                      use_pc;
 
+    genvar i;
+
     VX_generic_register #(
         .N(45 + `NW_BITS-1 + 1 + `NUM_THREADS*65)
     ) lsu_buffer (
@@ -49,17 +51,6 @@ module VX_lsu_unit #(
 
     wire core_req_rw = (use_mem_write != `BYTE_EN_NO);
 
-    reg [3:0] wmask;
-    always @(*) begin
-        case (use_mem_write[1:0])
-            0:        wmask = 4'b0001;   
-            1:        wmask = 4'b0011;
-            default : wmask = 4'b1111;
-        endcase
-    end
-
-    genvar i;
-
     wire [`NUM_THREADS-1:0][4:0]  mem_req_offset;
     wire [`NUM_THREADS-1:0][29:0] mem_req_addr;    
     wire [`NUM_THREADS-1:0][3:0]  mem_req_byteen;
@@ -68,25 +59,18 @@ module VX_lsu_unit #(
     wire [`NUM_THREADS-1:0][4:0]  mem_rsp_offset;
     wire[2:0] core_rsp_mem_read;
 
-    for (i = 0; i < `NUM_THREADS; ++i) begin  
-        always @(*) begin            
-            case (core_req_rw ? use_mem_write[1:0] : use_mem_read[1:0])
-                2'b0: begin 
-                    case (use_address[i][1:0])
-                        1:      mem_req_offset[i] = 8;
-                        2:      mem_req_offset[i] = 16;
-                        3:      mem_req_offset[i] = 24;    
-                       default: mem_req_offset[i] = 0;                   
-                    endcase
-                end
-                2'b1: begin
-                    mem_req_offset[i] = (2 == use_address[i][1:0]) ? 16 : 0;
-                end
-                default: mem_req_offset[i] = 0;                    
-            endcase
-        end
+    reg [3:0] wmask;
+    always @(*) begin
+        case ((core_req_rw ? use_mem_write[1:0] : use_mem_read[1:0]))
+            0:        wmask = 4'b0001;   
+            1:        wmask = 4'b0011;
+            default : wmask = 4'b1111;
+        endcase
+    end
 
+    for (i = 0; i < `NUM_THREADS; ++i) begin  
         assign mem_req_addr[i]   = use_address[i][31:2];        
+        assign mem_req_offset[i] = {3'b0, use_address[i][1:0]} << 3;
         assign mem_req_byteen[i] = (wmask << use_address[i][1:0]);
         assign mem_req_data[i]   = (use_store_data[i] << mem_req_offset[i]);
     end   
@@ -96,7 +80,9 @@ module VX_lsu_unit #(
     wire [`LOG2UP(`DCREQ_SIZE)-1:0] mrq_write_addr, mrq_read_addr, dbg_mrq_write_addr;
     wire mrq_full;
 
-    wire mrq_push     = (0 == core_req_rw) && (| dcache_req_if.core_req_valid) && dcache_req_if.core_req_ready;    
+    wire mrq_push = (| dcache_req_if.core_req_valid) && dcache_req_if.core_req_ready
+                 && (0 == core_req_rw); // only push read requests
+
     wire mrq_pop_part = (| dcache_rsp_if.core_rsp_valid) && dcache_rsp_if.core_rsp_ready;    
     
     assign mrq_read_addr = dcache_rsp_if.core_rsp_tag[0][`LOG2UP(`DCREQ_SIZE)-1:0];    
diff --git a/hw/rtl/VX_scheduler.v b/hw/rtl/VX_scheduler.v
index 921d7610..f036f293 100644
--- a/hw/rtl/VX_scheduler.v
+++ b/hw/rtl/VX_scheduler.v
@@ -59,6 +59,7 @@ module VX_scheduler (
                     rename_table[w][i] <= 0;
                 end
             end
+            count_valid <= 0;
         end else begin
             if (valid_wb) begin
                 assert(rename_table[writeback_if.warp_num][writeback_if.rd] != 0);
diff --git a/hw/rtl/Vortex.v b/hw/rtl/Vortex.v
index c4f9a8ca..6861eca8 100644
--- a/hw/rtl/Vortex.v
+++ b/hw/rtl/Vortex.v
@@ -271,7 +271,7 @@ module Vortex #(
     );
 
     // use "case equality" to handle uninitialized address value
-    wire io_select = ((dcache_io_core_req_if.core_req_addr[0] >= `BYTE_TO_WORD_ADDR(`IO_BUS_BASE_ADDR, `DWORD_SIZE)) === 1'b1);
+    wire io_select = (({dcache_io_core_req_if.core_req_addr[0], 2'b0} >= `IO_BUS_BASE_ADDR) === 1'b1);
 
     VX_dcache_io_arb dcache_io_arb (
         .io_select          (io_select),
diff --git a/hw/rtl/Vortex_Cluster.v b/hw/rtl/Vortex_Cluster.v
index 3c00f058..f90082dd 100644
--- a/hw/rtl/Vortex_Cluster.v
+++ b/hw/rtl/Vortex_Cluster.v
@@ -394,10 +394,11 @@ module Vortex_Cluster #(
         end
 
         VX_snp_forwarder #(
-            .BANK_LINE_SIZE(`L2BANK_LINE_SIZE), 
-            .NUM_REQUESTS(`NUM_CORES), 
-            .SNRQ_SIZE(`L2SNRQ_SIZE),
-            .SNP_REQ_TAG_WIDTH(`L2SNP_TAG_WIDTH)
+            .CACHE_ID           (`L2CACHE_ID),
+            .BANK_LINE_SIZE     (`L2BANK_LINE_SIZE), 
+            .NUM_REQUESTS       (`NUM_CORES), 
+            .SNRQ_SIZE          (`L2SNRQ_SIZE),
+            .SNP_REQ_TAG_WIDTH  (`L2SNP_TAG_WIDTH)
         ) snp_forwarder (
             .clk                (clk),
             .reset              (reset),
diff --git a/hw/rtl/Vortex_Socket.v b/hw/rtl/Vortex_Socket.v
index 329c3bc0..78382a5e 100644
--- a/hw/rtl/Vortex_Socket.v
+++ b/hw/rtl/Vortex_Socket.v
@@ -9,27 +9,27 @@ module Vortex_Socket (
     // DRAM request
     output wire                             dram_req_valid,
     output wire                             dram_req_rw,    
-    output wire[`L3DRAM_BYTEEN_WIDTH-1:0]   dram_req_byteen,    
-    output wire[`L3DRAM_ADDR_WIDTH-1:0]     dram_req_addr,
-    output wire[`L3DRAM_LINE_WIDTH-1:0]     dram_req_data,
-    output wire[`L3DRAM_TAG_WIDTH-1:0]      dram_req_tag,
+    output wire[`VX_DRAM_BYTEEN_WIDTH-1:0]  dram_req_byteen,    
+    output wire[`VX_DRAM_ADDR_WIDTH-1:0]    dram_req_addr,
+    output wire[`VX_DRAM_LINE_WIDTH-1:0]    dram_req_data,
+    output wire[`VX_DRAM_TAG_WIDTH-1:0]     dram_req_tag,
     input  wire                             dram_req_ready,
 
     // DRAM response    
     input wire                              dram_rsp_valid,        
-    input wire[`L3DRAM_LINE_WIDTH-1:0]      dram_rsp_data,
-    input wire[`L3DRAM_TAG_WIDTH-1:0]       dram_rsp_tag,
+    input wire[`VX_DRAM_LINE_WIDTH-1:0]     dram_rsp_data,
+    input wire[`VX_DRAM_TAG_WIDTH-1:0]      dram_rsp_tag,
     output wire                             dram_rsp_ready,
 
     // Snoop request
     input wire                              snp_req_valid,
-    input wire[`L3DRAM_ADDR_WIDTH-1:0]      snp_req_addr,
-    input wire[`L3SNP_TAG_WIDTH-1:0]        snp_req_tag,
+    input wire[`VX_DRAM_ADDR_WIDTH-1:0]     snp_req_addr,
+    input wire[`VX_SNP_TAG_WIDTH-1:0]       snp_req_tag,
     output wire                             snp_req_ready, 
 
     // Snoop response
     output wire                             snp_rsp_valid,
-    output wire[`L3SNP_TAG_WIDTH-1:0]       snp_rsp_tag,
+    output wire[`VX_SNP_TAG_WIDTH-1:0]      snp_rsp_tag,
     input wire                              snp_rsp_ready,     
 
     // I/O request
@@ -38,13 +38,13 @@ module Vortex_Socket (
     output wire[3:0]                        io_req_byteen,  
     output wire[29:0]                       io_req_addr,
     output wire[31:0]                       io_req_data,    
-    output wire[`DCORE_TAG_WIDTH-1:0]       io_req_tag,    
+    output wire[`VX_CORE_TAG_WIDTH-1:0]     io_req_tag,    
     input wire                              io_req_ready,
 
     // I/O response
     input wire                              io_rsp_valid,
     input wire[31:0]                        io_rsp_data,
-    input wire[`DCORE_TAG_WIDTH-1:0]        io_rsp_tag,
+    input wire[`VX_CORE_TAG_WIDTH-1:0]      io_rsp_tag,
     output wire                             io_rsp_ready,
 
     // Status
@@ -334,7 +334,7 @@ module Vortex_Socket (
 `ifdef DBG_PRINT_DRAM
     always_ff @(posedge clk) begin
         if (dram_req_valid && dram_req_ready) begin
-            $display("%t: DRAM req: rw=%b addr=%0h, tag=%0h, byteen=%0h data=%0h", $time, dram_req_rw, {dram_req_addr, `CLOG2(`GLOBAL_BLOCK_SIZE)'(0)}, dram_req_tag, dram_req_byteen, dram_req_data);
+            $display("%t: DRAM req: rw=%b addr=%0h, tag=%0h, byteen=%0h data=%0h", $time, dram_req_rw, `DRAM_TO_BYTE_ADDR(dram_req_addr), dram_req_tag, dram_req_byteen, dram_req_data);
         end
         if (dram_rsp_valid && dram_rsp_ready) begin
             $display("%t: DRAM rsp: tag=%0h, data=%0h", $time, dram_rsp_tag, dram_rsp_data);
diff --git a/hw/rtl/cache/VX_cache.v b/hw/rtl/cache/VX_cache.v
index 3209e6fe..64e31d28 100644
--- a/hw/rtl/cache/VX_cache.v
+++ b/hw/rtl/cache/VX_cache.v
@@ -169,6 +169,7 @@ module VX_cache #(
 
     if (SNOOP_FORWARDING) begin
         VX_snp_forwarder #(
+            .CACHE_ID          (CACHE_ID),
             .BANK_LINE_SIZE    (BANK_LINE_SIZE), 
             .NUM_REQUESTS      (NUM_SNP_REQUESTS), 
             .SNRQ_SIZE         (SNRQ_SIZE),
diff --git a/hw/rtl/cache/VX_cache_config.vh b/hw/rtl/cache/VX_cache_config.vh
index 930e50f3..52af265e 100644
--- a/hw/rtl/cache/VX_cache_config.vh
+++ b/hw/rtl/cache/VX_cache_config.vh
@@ -72,8 +72,6 @@
 
 `define DRAM_TO_LINE_ADDR(x)    x[`DRAM_ADDR_WIDTH-1:`BANK_SELECT_BITS]
 
-`define BYTE_TO_WORD_ADDR(x, w) (32-`CLOG2(w))'(x >> `CLOG2(w)) 
-
 `define LINE_TO_DRAM_ADDR(x, i) {x, `BANK_SELECT_BITS'(i)}
 
 `define LINE_TO_BYTE_ADDR(x, i) {x, `BANK_SELECT_BITS'(i), `BASE_ADDR_BITS'(0)}
diff --git a/hw/rtl/cache/VX_snp_forwarder.v b/hw/rtl/cache/VX_snp_forwarder.v
index 688db369..2d8570dc 100644
--- a/hw/rtl/cache/VX_snp_forwarder.v
+++ b/hw/rtl/cache/VX_snp_forwarder.v
@@ -1,6 +1,7 @@
 `include "VX_cache_config.vh"
 
 module VX_snp_forwarder #(
+    parameter CACHE_ID          = 0, 
     parameter BANK_LINE_SIZE    = 0, 
     parameter NUM_REQUESTS      = 0, 
     parameter SNRQ_SIZE         = 0,
@@ -111,16 +112,16 @@ module VX_snp_forwarder #(
 `ifdef DBG_PRINT_CACHE_SNP
      always_ff @(posedge clk) begin
         if (snp_req_valid && snp_req_ready) begin
-            $display("%t: snp req: addr=%0h, tag=%0h", $time, {snp_req_addr, `LOG2UP(BANK_LINE_SIZE)'(0)}, snp_req_tag);
+            $display("%t: cache%01d snp req: addr=%0h, tag=%0h", $time, CACHE_ID, `DRAM_TO_BYTE_ADDR(snp_req_addr), snp_req_tag);
         end
         if (snp_fwdout_valid[0] && snp_fwdout_ready[0]) begin
-            $display("%t: snp fwd_out: addr=%0h, tag=%0h", $time, {snp_fwdout_addr[0], `LOG2UP(BANK_LINE_SIZE)'(0)}, snp_fwdout_tag[0]);
+            $display("%t: cache%01d snp fwd_out: addr=%0h, tag=%0h", $time, CACHE_ID, `DRAM_TO_BYTE_ADDR(snp_fwdout_addr[0]), snp_fwdout_tag[0]);
         end
         if (fwdin_valid && fwdin_ready) begin
-            $display("%t: snp fwd_in[%01d]: tag=%0h", $time, fwdin_sel, fwdin_tag);
+            $display("%t: cache%01d snp fwd_in[%01d]: tag=%0h", $time, CACHE_ID, fwdin_sel, fwdin_tag);
         end
         if (snp_rsp_valid && snp_rsp_ready) begin
-            $display("%t: snp rsp: addr=%0h, tag=%0h", $time, snp_rsp_addr, snp_rsp_tag);
+            $display("%t: cache%01d snp rsp: addr=%0h, tag=%0h", $time, CACHE_ID, snp_rsp_addr, snp_rsp_tag);
         end
     end
 `endif
diff --git a/hw/rtl/cache/VX_tag_data_structure.v b/hw/rtl/cache/VX_tag_data_structure.v
index 4d4f87ab..b76ef266 100644
--- a/hw/rtl/cache/VX_tag_data_structure.v
+++ b/hw/rtl/cache/VX_tag_data_structure.v
@@ -30,12 +30,12 @@ module VX_tag_data_structure #(
     input  wire                             fill_sent    
 );
 
-    reg [`BANK_LINE_WORDS-1:0][WORD_SIZE-1:0][7:0] data [`BANK_LINE_COUNT-1:0];
-    reg [`BANK_LINE_WORDS-1:0][WORD_SIZE-1:0]    dirtyb [`BANK_LINE_COUNT-1:0];
+    reg [`BANK_LINE_WORDS-1:0][WORD_SIZE-1:0][7:0] data [`BANK_LINE_COUNT-1:0];    
     reg [`TAG_SELECT_BITS-1:0]                      tag [`BANK_LINE_COUNT-1:0];
-    reg                                           valid [`BANK_LINE_COUNT-1:0];
-    reg                                           dirty [`BANK_LINE_COUNT-1:0];   
-
+    reg [`BANK_LINE_WORDS-1:0][WORD_SIZE-1:0]     dirtyb[`BANK_LINE_COUNT-1:0];
+    reg                                            dirty[`BANK_LINE_COUNT-1:0];   
+    reg [`BANK_LINE_COUNT-1:0]                     valid;    
+    
     assign read_valid  = valid  [read_addr];
     assign read_dirty  = dirty  [read_addr];
     assign read_dirtyb = dirtyb [read_addr];
@@ -48,9 +48,7 @@ module VX_tag_data_structure #(
     always @(posedge clk) begin
         if (reset) begin
             for (i = 0; i < `BANK_LINE_COUNT; i++) begin
-                valid[i]  <= 0;
-                dirty[i]  <= 0;
-                dirtyb[i] <= 0;
+                valid[i] <= 0;
             end
         end else if (!stall_bank_pipe) begin
             if (do_write) begin
diff --git a/hw/syn/quartus/top/Makefile b/hw/syn/quartus/top/Makefile
index 370d7320..62b5cdd9 100644
--- a/hw/syn/quartus/top/Makefile
+++ b/hw/syn/quartus/top/Makefile
@@ -1,6 +1,6 @@
-PROJECT = Vortex
-TOP_LEVEL_ENTITY = Vortex
-SRC_FILE = Vortex.v
+PROJECT = Vortex_Socket
+TOP_LEVEL_ENTITY = Vortex_Socket
+SRC_FILE = Vortex_Socket.v
 PROJECT_FILES = $(PROJECT).qpf $(PROJECT).qsf
 
 # Part, Family
diff --git a/hw/syn/quartus/vortex/Makefile b/hw/syn/quartus/vortex/Makefile
new file mode 100644
index 00000000..370d7320
--- /dev/null
+++ b/hw/syn/quartus/vortex/Makefile
@@ -0,0 +1,70 @@
+PROJECT = Vortex
+TOP_LEVEL_ENTITY = Vortex
+SRC_FILE = Vortex.v
+PROJECT_FILES = $(PROJECT).qpf $(PROJECT).qsf
+
+# Part, Family
+FAMILY = "Arria 10"
+DEVICE = 10AX115N3F40E2SG   
+
+# Executable Configuration
+SYN_ARGS = --parallel --read_settings_files=on
+FIT_ARGS = --part=$(DEVICE) --read_settings_files=on
+ASM_ARGS =
+STA_ARGS = --do_report_timing
+
+# Build targets
+all: $(PROJECT).sta.rpt
+
+syn: $(PROJECT).syn.rpt
+
+fit: $(PROJECT).fit.rpt
+
+asm: $(PROJECT).asm.rpt
+
+sta: $(PROJECT).sta.rpt
+
+smart: smart.log
+
+# Target implementations
+STAMP = echo done >
+
+$(PROJECT).syn.rpt: smart.log syn.chg $(SOURCE_FILES) 
+	quartus_syn $(PROJECT) $(SYN_ARGS)
+	$(STAMP) fit.chg
+
+$(PROJECT).fit.rpt: smart.log fit.chg $(PROJECT).syn.rpt
+	quartus_fit $(PROJECT) $(FIT_ARGS)
+	$(STAMP) asm.chg
+	$(STAMP) sta.chg
+
+$(PROJECT).asm.rpt: smart.log asm.chg $(PROJECT).fit.rpt
+	quartus_asm $(PROJECT) $(ASM_ARGS)
+
+$(PROJECT).sta.rpt: smart.log sta.chg $(PROJECT).fit.rpt
+	quartus_sta $(PROJECT)  $(STA_ARGS) 
+
+smart.log: $(PROJECT_FILES)
+	quartus_sh --determine_smart_action $(PROJECT) > smart.log
+
+# Project initialization
+$(PROJECT_FILES):
+	quartus_sh -t project.tcl -project $(PROJECT) -family $(FAMILY) -device $(DEVICE) -top $(TOP_LEVEL_ENTITY) -src $(SRC_FILE) -sdc project.sdc -inc "../../../rtl;../../../rtl/libs;../../../rtl/interfaces;../../../rtl/pipe_regs;../../../rtl/cache"
+	
+syn.chg:
+	$(STAMP) syn.chg
+
+fit.chg:
+	$(STAMP) fit.chg
+
+sta.chg:
+	$(STAMP) sta.chg
+
+asm.chg:
+	$(STAMP) asm.chg
+
+program: $(PROJECT).sof
+	quartus_pgm --no_banner --mode=jtag -o "$(PROJECT).sof"
+	
+clean:
+	rm -rf bin *.rpt *.chg *.qsf *.qpf *.qws smart.log *.htm *.eqn *.pin *.sof *.pof qdb incremental_db tmp-clearbox
diff --git a/hw/syn/quartus/vortex/project.tcl b/hw/syn/quartus/vortex/project.tcl
new file mode 100644
index 00000000..afe69d48
--- /dev/null
+++ b/hw/syn/quartus/vortex/project.tcl
@@ -0,0 +1,41 @@
+load_package flow
+package require cmdline
+
+set options { \
+    { "project.arg" "" "Project name" } \
+    { "family.arg" "" "Device family name" } \
+    { "device.arg" "" "Device name" } \
+    { "top.arg" "" "Top level module" } \
+    { "sdc.arg" "" "Timing Design Constraints file" } \
+    { "src.arg" "" "Verilog source file" } \
+    { "inc.arg" "." "Include path" } \
+}
+
+array set opts [::cmdline::getoptions quartus(args) $options]
+
+project_new $opts(project) -overwrite
+
+set_global_assignment -name FAMILY $opts(family)
+set_global_assignment -name DEVICE $opts(device)
+set_global_assignment -name TOP_LEVEL_ENTITY $opts(top)
+set_global_assignment -name VERILOG_FILE $opts(src)
+set_global_assignment -name SEARCH_PATH $opts(inc)
+set_global_assignment -name SDC_FILE $opts(sdc)
+set_global_assignment -name PROJECT_OUTPUT_DIRECTORY bin
+set_global_assignment -name NUM_PARALLEL_PROCESSORS ALL
+set_global_assignment -name VERILOG_INPUT_VERSION SYSTEMVERILOG_2009
+
+proc make_all_pins_virtual {} {
+    execute_module -tool map
+    set name_ids [get_names -filter * -node_type pin]
+    foreach_in_collection name_id $name_ids {        
+        set pin_name [get_name_info -info full_path $name_id]
+        post_message "Making VIRTUAL_PIN assignment to $pin_name"
+        set_instance_assignment -to $pin_name -name VIRTUAL_PIN ON
+    }
+    export_assignments
+}
+
+make_all_pins_virtual
+
+project_close
\ No newline at end of file
diff --git a/hw/syn/quartus/top/VX_timing.tcl b/hw/syn/quartus/vortex/timing.tcl
similarity index 100%
rename from hw/syn/quartus/top/VX_timing.tcl
rename to hw/syn/quartus/vortex/timing.tcl
diff --git a/hw/syn/quartus/vortex/vortex.sdc b/hw/syn/quartus/vortex/vortex.sdc
new file mode 100644
index 00000000..3c588f3b
--- /dev/null
+++ b/hw/syn/quartus/vortex/vortex.sdc
@@ -0,0 +1,9 @@
+set_time_format -unit ns -decimal_places 3
+
+create_clock -name {clk} -period "250 MHz" -waveform { 0.0 1.0 } [get_ports {clk}]
+
+derive_pll_clocks -create_base_clocks
+derive_clock_uncertainty
+
+
+