From 9575fe9a519d55611318a8169a1954a9c423e988 Mon Sep 17 00:00:00 2001 From: Blaise Tine Date: Mon, 8 Jun 2020 06:54:47 -0700 Subject: [PATCH] scope fixes --- driver/opae/vortex.cpp | 356 ++++++++++++++++++++--------------------- driver/rtlsim/Makefile | 2 +- hw/opae/sources.txt | 1 + hw/opae/vortex_afu.sv | 3 +- hw/rtl/VX_define.vh | 14 +- hw/rtl/libs/VX_scope.v | 76 ++++++--- 6 files changed, 246 insertions(+), 206 deletions(-) diff --git a/driver/opae/vortex.cpp b/driver/opae/vortex.cpp index 75d17c8d..7a7d23a1 100755 --- a/driver/opae/vortex.cpp +++ b/driver/opae/vortex.cpp @@ -65,9 +65,179 @@ inline bool is_aligned(size_t addr, size_t alignment) { return 0 == (addr & (alignment - 1)); } -#ifdef SCOPE -std::future future_scope; -#endif +/////////////////////////////////////////////////////////////////////////////// + +static int vx_scope_trace(vx_device_h hdevice) { + if (nullptr == hdevice) + return -1; + + vx_device_t *device = ((vx_device_t*)hdevice); + + std::ofstream ofs("vx_scope.vcd"); + + ofs << "$timescale 1 ns $end" << std::endl; + + int fwidth = 0; + + ofs << "$var reg 1 0 clk $end" << std::endl; + + fwidth += 1; + + ofs << "$var reg 1 1 icache_req_valid $end" << std::endl; + ofs << "$var reg 1 2 icache_req_ready $end" << std::endl; + ofs << "$var reg 1 3 icache_rsp_valid $end" << std::endl; + ofs << "$var reg 1 4 icache_rsp_ready $end" << std::endl; + ofs << "$var reg 4 5 dcache_req_valid $end" << std::endl; + ofs << "$var reg 1 6 dcache_req_ready $end" << std::endl; + ofs << "$var reg 4 7 dcache_rsp_valid $end" << std::endl; + ofs << "$var reg 1 8 dcache_rsp_ready $end" << std::endl; + ofs << "$var reg 1 9 dram_req_valid $end" << std::endl; + ofs << "$var reg 1 10 dram_req_ready $end" << std::endl; + ofs << "$var reg 1 11 dram_rsp_valid $end" << std::endl; + ofs << "$var reg 1 12 dram_rsp_ready $end" << std::endl; + ofs << "$var reg 1 13 schedule_delay $end" << std::endl; + + fwidth += 19; + + ofs << "$var reg 2 14 icache_req_tag $end" << std::endl; + ofs << "$var reg 2 15 icache_rsp_tag $end" << std::endl; + ofs << "$var reg 2 16 dcache_req_tag $end" << std::endl; + ofs << "$var reg 2 17 dcache_rsp_tag $end" << std::endl; + ofs << "$var reg 29 18 dram_req_tag $end" << std::endl; + ofs << "$var reg 29 19 dram_rsp_tag $end" << std::endl; + + fwidth += 66; + + const int num_signals = 20; + + ofs << "enddefinitions $end" << std::endl; + + uint64_t frame_width, max_frames, data_valid; + + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 2)); + CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &frame_width)); + std::cout << "scope::frame_width=" << frame_width << std::endl; + + assert((fwidth-1)== (int)frame_width); + + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 3)); + CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &max_frames)); + std::cout << "scope::max_frames=" << max_frames << std::endl; + + do { + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 0)); + CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &data_valid)); + if (data_valid) + break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (true); + + std::cout << "scope trace dump begin..." << std::endl; + + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 1)); + + std::vector signa_data(frame_width+1); + + uint64_t frame_offset = 0, frame_no = 0, timestamp = 0; + + int signal_id = 0; + int signal_offset = 0; + + auto print_header = [&] () { + ofs << '#' << timestamp++ << std::endl; + ofs << "b0 0" << std::endl; + ofs << '#' << timestamp++ << std::endl; + ofs << "b1 0" << std::endl; + + uint64_t delta; + fpga_result res = fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &delta); + assert(res == FPGA_OK); + + while (delta != 0) { + ofs << '#' << timestamp++ << std::endl; + ofs << "b0 0" << std::endl; + ofs << '#' << timestamp++ << std::endl; + ofs << "b1 0" << std::endl; + --delta; + } + + signal_id = 1; + }; + + auto print_signal = [&] (uint64_t word, int signal_width) { + + int word_offset = frame_offset % 64; + + signa_data[signal_width - signal_offset - 1] = ((word >> word_offset) & 0x1) ? '1' : '0'; + + ++signal_offset; + ++frame_offset; + + if (signal_offset == signal_width) { + signa_data[signal_width] = 0; // string null termination + ofs << 'b' << signa_data.data() << ' ' << (num_signals - signal_id) << std::endl; + signal_offset = 0; + ++signal_id; + } + + if (frame_offset == frame_width) { + assert(0 == signal_offset); + signal_id = 0; + frame_offset = 0; + ++frame_no; + if (frame_no != max_frames) { + print_header(); + } + } + }; + + print_header(); + + do { + if (frame_no == max_frames-1) { + // verify last frame is valid + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 0)); + CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &data_valid)); + assert(data_valid == 1); + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 1)); + } + + uint64_t word; + CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &word)); + + do { + switch (num_signals - signal_id) { + case 14: + case 15: + case 16: + case 17: + print_signal(word, 2); + break; + case 5: + case 7: + print_signal(word, 4); + break; + case 18: + case 19: + print_signal(word, 29); + break; + default: + print_signal(word, 1); + break; + } + } while ((frame_offset % 64) != 0); + + } while (frame_no != max_frames); + + std::cout << "scope trace dump done!" << std::endl; + + // verify data not valid + CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 0)); + CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &data_valid)); + assert(data_valid == 0); + + return 0; +} /////////////////////////////////////////////////////////////////////////////// @@ -134,10 +304,6 @@ extern int vx_dev_close(vx_device_h hdevice) { if (nullptr == hdevice) return -1; -#ifdef SCOPE - future_scope.wait(); -#endif - vx_device_t *device = ((vx_device_t*)hdevice); fpgaClose(device->fpga); @@ -393,171 +559,6 @@ extern int vx_flush_caches(vx_device_h hdevice, size_t dev_maddr, size_t size) { return 0; } -static int vx_scope_trace(vx_device_h hdevice) { - if (nullptr == hdevice) - return -1; - - vx_device_t *device = ((vx_device_t*)hdevice); - - std::ofstream ofs("vx_scope.vcd"); - - ofs << "$timescale 1 ns $end" << std::endl; - - int fwidth = 0; - - ofs << "$var reg 1 0 clk $end" << std::endl; - - fwidth += 1; - - ofs << "$var reg 1 1 icache_req_valid $end" << std::endl; - ofs << "$var reg 2 2 icache_req_tag $end" << std::endl; - ofs << "$var reg 1 3 icache_req_ready $end" << std::endl; - ofs << "$var reg 1 4 icache_rsp_valid $end" << std::endl; - ofs << "$var reg 2 5 icache_rsp_tag $end" << std::endl; - ofs << "$var reg 1 6 icache_rsp_ready $end" << std::endl; - - fwidth += 8; - - ofs << "$var reg 4 7 dcache_req_valid $end" << std::endl; - ofs << "$var reg 2 8 dcache_req_tag $end" << std::endl; - ofs << "$var reg 1 9 dcache_req_ready $end" << std::endl; - ofs << "$var reg 4 10 dcache_rsp_valid $end" << std::endl; - ofs << "$var reg 2 11 dcache_rsp_tag $end" << std::endl; - ofs << "$var reg 1 12 dcache_rsp_ready $end" << std::endl; - - fwidth += 14; - - ofs << "$var reg 1 13 dram_req_valid $end" << std::endl; - ofs << "$var reg 29 14 dram_req_tag $end" << std::endl; - ofs << "$var reg 1 15 dram_req_ready $end" << std::endl; - ofs << "$var reg 1 16 dram_rsp_valid $end" << std::endl; - ofs << "$var reg 29 17 dram_rsp_tag $end" << std::endl; - ofs << "$var reg 1 18 dram_rsp_ready $end" << std::endl; - - fwidth += 62; - - ofs << "$var reg 1 19 schedule_delay $end" << std::endl; - - fwidth += 1; - - const int num_signals = 20; - - ofs << "enddefinitions $end" << std::endl; - - uint64_t frame_width, max_frames, data_valid; - - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 2)); - CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &frame_width)); - std::cout << "scope::frame_width=" << frame_width << std::endl; - - assert((fwidth-1)== (int)frame_width); - - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 3)); - CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &max_frames)); - std::cout << "scope::max_frames=" << max_frames << std::endl; - - do { - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 0)); - CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &data_valid)); - if (data_valid) - break; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } while (true); - - std::cout << "scope trace dump begin..." << std::endl; - - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 1)); - - std::vector signa_data(frame_width+1); - - uint64_t frame_offset = 0, frame_no = 0; - - int signal_id = 0; - int signal_offset = 0; - - auto print_signal = [&] (uint64_t word, int signal_width) { - - int word_offset = frame_offset % 64; - - signa_data[signal_width - signal_offset - 1] = ((word >> word_offset) & 0x1) ? '1' : '0'; - - ++signal_offset; - ++frame_offset; - - if (signal_offset == signal_width) { - signa_data[signal_width] = 0; // string null termination - ofs << 'b' << signa_data.data() << ' ' << (num_signals - signal_id) << std::endl; - signal_offset = 0; - ++signal_id; - } - - if (frame_offset == frame_width) { - assert(0 == signal_offset); - signal_id = 0; - frame_offset = 0; - ++frame_no; - if (frame_no != max_frames) { - ofs << '#' << (frame_no * 2 + 0) << std::endl; - ofs << "b0 0" << std::endl; - ofs << '#' << (frame_no * 2 + 1) << std::endl; - ofs << "b1 0" << std::endl; - ++signal_id; - } - } - }; - - ofs << '#' << (frame_no * 2 + 0) << std::endl; - ofs << "b0 0" << std::endl; - ofs << '#' << (frame_no * 2 + 1) << std::endl; - ofs << "b1 0" << std::endl; - ++signal_id; - - do { - if (frame_no == max_frames-1) { - // verify last frame is valid - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 0)); - CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &data_valid)); - assert(data_valid == 1); - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 1)); - } - - uint64_t word; - CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &word)); - - do { - switch (num_signals - signal_id) { - case 2: - case 5: - case 8: - case 11: - print_signal(word, 2); - break; - case 7: - case 10: - print_signal(word, 4); - break; - case 14: - case 17: - print_signal(word, 29); - break; - default: - print_signal(word, 1); - break; - } - } while ((frame_offset % 64) != 0); - - } while (frame_no != max_frames); - - std::cout << "scope trace dump done!" << std::endl; - - // verify data not valid - CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_CMD, 0)); - CHECK_RES(fpgaReadMMIO64(device->fpga, 0, MMIO_CSR_SCOPE_DATA, &data_valid)); - assert(data_valid == 0); - - return 0; -} - extern int vx_start(vx_device_h hdevice) { if (nullptr == hdevice) return -1; @@ -566,15 +567,14 @@ extern int vx_start(vx_device_h hdevice) { // Ensure ready for new command if (vx_ready_wait(hdevice, -1) != 0) - return -1; - -#ifdef SCOPE - // launch logic scope - future_scope = std::async(std::launch::async, [&]{ vx_scope_trace(hdevice); }); -#endif + return -1; // start execution CHECK_RES(fpgaWriteMMIO64(device->fpga, 0, MMIO_CSR_CMD, CMD_TYPE_RUN)); +#ifdef SCOPE + vx_scope_trace(hdevice); +#endif + return 0; } \ No newline at end of file diff --git a/driver/rtlsim/Makefile b/driver/rtlsim/Makefile index 0d6df196..35c0aa17 100644 --- a/driver/rtlsim/Makefile +++ b/driver/rtlsim/Makefile @@ -16,7 +16,7 @@ DBG_PRINT_FLAGS = -DDBG_PRINT_CORE_ICACHE \ #MULTICORE += -DNUM_CLUSTERS=2 -DNUM_CORES=4 #MULTICORE += -DNUM_CLUSTERS=1 -DNUM_CORES=4 -MULTICORE += -DNUM_CLUSTERS=1 -DNUM_CORES=2 +#MULTICORE += -DNUM_CLUSTERS=1 -DNUM_CORES=2 #DEBUG=1 AFU=1 diff --git a/hw/opae/sources.txt b/hw/opae/sources.txt index 8481da68..bd5ac762 100644 --- a/hw/opae/sources.txt +++ b/hw/opae/sources.txt @@ -9,6 +9,7 @@ vortex_afu.json +define+DBG_PRINT_CACHE_MSRQ +define+DBG_PRINT_DRAM +define+DBG_PRINT_OPAE ++define+DBG_PRINT_SCOPE +define+SCOPE diff --git a/hw/opae/vortex_afu.sv b/hw/opae/vortex_afu.sv index 6795583f..e23b64e0 100644 --- a/hw/opae/vortex_afu.sv +++ b/hw/opae/vortex_afu.sv @@ -802,7 +802,8 @@ end VX_scope #( .DATAW ($bits({`SCOPE_SIGNALS_LIST})), .BUSW (64), - .SIZE (256) + .SIZE (256), + .IDW (19) ) scope ( .clk (clk), .reset (SoftReset), diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index 4cc8b8fc..d6843487 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -285,24 +285,24 @@ `ifdef SCOPE `define SCOPE_SIGNALS_LIST \ scope_icache_req_valid, \ - scope_icache_req_tag, \ scope_icache_req_ready, \ scope_icache_rsp_valid, \ - scope_icache_rsp_tag, \ scope_icache_rsp_ready, \ scope_dcache_req_valid, \ - scope_dcache_req_tag, \ scope_dcache_req_ready, \ scope_dcache_rsp_valid, \ - scope_dcache_rsp_tag, \ scope_dcache_rsp_ready, \ scope_dram_req_valid, \ - scope_dram_req_tag, \ scope_dram_req_ready, \ scope_dram_rsp_valid, \ - scope_dram_rsp_tag, \ scope_dram_rsp_ready, \ - scope_schedule_delay + scope_schedule_delay, \ + scope_icache_req_tag, \ + scope_icache_rsp_tag, \ + scope_dcache_req_tag, \ + scope_dcache_rsp_tag, \ + scope_dram_req_tag, \ + scope_dram_rsp_tag `define SCOPE_SIGNALS_DECL \ wire scope_icache_req_valid; \ diff --git a/hw/rtl/libs/VX_scope.v b/hw/rtl/libs/VX_scope.v index 428c5057..2bbe70e3 100644 --- a/hw/rtl/libs/VX_scope.v +++ b/hw/rtl/libs/VX_scope.v @@ -3,7 +3,8 @@ module VX_scope #( parameter DATAW = 64, parameter BUSW = 64, - parameter SIZE = 256 + parameter SIZE = 256, + parameter IDW = 1 ) ( input wire clk, input wire reset, @@ -14,6 +15,8 @@ module VX_scope #( input wire bus_write, input wire bus_read ); + localparam DELTA_ENABLE = (IDW != 0); + typedef enum logic[2:0] { CMD_GET_VALID, CMD_GET_DATA, @@ -32,14 +35,17 @@ module VX_scope #( GET_DEPTH } cmd_get_t; - reg [DATAW-1:0] mem [SIZE-1:0]; - //reg [63:0] offsets [SIZE-1:0]; + reg [DATAW-1:0] data_store [SIZE-1:0]; + + reg [63:0] delta_store [SIZE-1:0]; + reg [IDW-1:0] prev_id; + reg [63:0] delta; reg [`CLOG2(SIZE)-1:0] raddr, waddr, waddr_end; reg [`LOG2UP(DATAW)-1:0] read_offset; - reg start_wait, recording, data_valid; + reg start_wait, recording, data_valid, read_delta; reg [BUSW-3:0] delay_val, delay_cntr; @@ -49,6 +55,8 @@ module VX_scope #( wire [BUSW-4:0] cmd_data; assign {cmd_data, cmd_type} = bus_in; + wire [IDW-1:0] trigger_id = data_in[DATAW-1:DATAW-IDW]; + always @(posedge clk) begin if (reset) begin raddr <= 0; @@ -61,6 +69,8 @@ module VX_scope #( out_cmd <= $bits(out_cmd)'(CMD_GET_VALID); delay_val <= 0; waddr_end <= $bits(waddr)'(SIZE-1); + delta <= 0; + read_delta <= 0; end else begin if (bus_write) begin @@ -80,7 +90,8 @@ module VX_scope #( if (0 == delay_val) begin start_wait <= 0; recording <= 1; - delay_cntr <= 0; + delay_cntr <= 0; + delta <= 0; end else begin start_wait <= 1; recording <= 0; @@ -93,35 +104,57 @@ module VX_scope #( if (1 == delay_cntr) begin start_wait <= 0; recording <= 1; + delta <= 0; end end if (recording) begin - mem[waddr] <= data_in; - waddr <= waddr + 1; + if (DELTA_ENABLE) begin + if (0 == waddr + || (trigger_id != prev_id)) begin + data_store[waddr] <= data_in; + delta_store[waddr] <= delta; + waddr <= waddr + 1; + delta <= 0; + end else begin + delta <= delta + 1; + end + prev_id <= trigger_id; + end else begin + data_store[waddr] <= data_in; + waddr <= waddr + 1; + end + if (waddr == waddr_end) begin recording <= 0; data_valid <= 1; + read_delta <= DELTA_ENABLE; end end if (bus_read && (out_cmd == GET_DATA) && data_valid) begin - if (DATAW > BUSW) begin - if (read_offset < $bits(read_offset)'(DATAW-BUSW)) begin - read_offset <= read_offset + $bits(read_offset)'(BUSW); + if (read_delta) begin + read_delta <= 0; + end else begin + if (DATAW > BUSW) begin + if (read_offset < $bits(read_offset)'(DATAW-BUSW)) begin + read_offset <= read_offset + $bits(read_offset)'(BUSW); + end else begin + raddr <= raddr + 1; + read_offset <= 0; + read_delta <= DELTA_ENABLE; + if (raddr == waddr_end) begin + data_valid <= 0; + end + end end else begin - raddr <= raddr + 1; - read_offset <= 0; + raddr <= raddr + 1; + read_delta <= DELTA_ENABLE; if (raddr == waddr_end) begin data_valid <= 0; end - end - end else begin - raddr <= raddr + 1; - if (raddr == waddr_end) begin - data_valid <= 0; end end end @@ -133,14 +166,19 @@ module VX_scope #( GET_VALID : bus_out = BUSW'(data_valid); GET_WIDTH : bus_out = BUSW'(DATAW); GET_DEPTH : bus_out = BUSW'(waddr_end) + BUSW'(1); - default : bus_out = (BUSW)'(mem[raddr] >> read_offset); + default : bus_out = read_delta ? (BUSW)'(delta_store[raddr]) : (BUSW)'(data_store[raddr] >> read_offset); endcase end +`ifdef DBG_PRINT_SCOPE always_ff @(posedge clk) begin if (bus_read) begin - $display("%t: read: cmd=%0d, out=0x%0h, addr=%0d, off=%0d", $time, out_cmd, bus_out, raddr, read_offset); + $display("%t: scope-read: cmd=%0d, out=0x%0h, addr=%0d, off=%0d", $time, out_cmd, bus_out, raddr, read_offset); + end + if (DELTA_ENABLE && recording && (trigger_id != prev_id) && (delta != 0)) begin + $display("%t: scope-write: waddr=%0d, delta=%0d", $time, waddr, delta); end end +`endif endmodule \ No newline at end of file