From e33102ac0865944998d1b4633a7444a9b5e14352 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 2 Feb 2023 13:25:19 -0800 Subject: [PATCH] Add spike-based cosim --- .../src/main/resources/csrc/cospike.cc | 241 ++++++++++++++++++ .../src/main/resources/vsrc/cospike.v | 78 ++++++ .../chipyard/src/main/scala/Cospike.scala | 104 ++++++++ .../chipyard/src/main/scala/DigitalTop.scala | 1 + .../src/main/scala/HarnessBinders.scala | 9 +- .../src/main/scala/config/BoomConfigs.scala | 6 + 6 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 generators/chipyard/src/main/resources/csrc/cospike.cc create mode 100644 generators/chipyard/src/main/resources/vsrc/cospike.v create mode 100644 generators/chipyard/src/main/scala/Cospike.scala diff --git a/generators/chipyard/src/main/resources/csrc/cospike.cc b/generators/chipyard/src/main/resources/csrc/cospike.cc new file mode 100644 index 00000000..2f6e7490 --- /dev/null +++ b/generators/chipyard/src/main/resources/csrc/cospike.cc @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include +#include + +typedef struct system_info_t { + std::string isa; + int pmpregions; + uint64_t mem0_base; + uint64_t mem0_size; + int nharts; + std::vector bootrom; +}; + +system_info_t* info = NULL; +sim_t* sim = NULL; +reg_t tohost_addr = 0; +reg_t fromhost_addr = 0; +std::set magic_addrs; +cfg_t* cfg; + +static std::vector> make_mems(const std::vector &layout) +{ + std::vector> mems; + mems.reserve(layout.size()); + for (const auto &cfg : layout) { + mems.push_back(std::make_pair(cfg.get_base(), new mem_t(cfg.get_size()))); + } + return mems; +} + +extern "C" void cospike_set_sysinfo(char* isa, int pmpregions, + long long int mem0_base, long long int mem0_size, + int nharts, + char* bootrom + ) { + if (!info) { + info = new system_info_t; + info->isa = std::string(isa); + info->pmpregions = pmpregions; + info->mem0_base = mem0_base; + info->mem0_size = mem0_size; + info->nharts = nharts; + std::stringstream ss(bootrom); + std::string s; + while (ss >> s) { + info->bootrom.push_back(std::stoi(s)); + } + } +} + +extern "C" void cospike_cosim(long long int cycle, + long long int hartid, + int has_wdata, + int valid, + long long int iaddr, + unsigned long int insn, + int exception, + int interrupt, + unsigned long long int cause, + unsigned long long int wdata) +{ + assert(info); + if (!sim) { + printf("Configuring spike cosim\n"); + std::vector mem_cfg; + std::vector hartids; + mem_cfg.push_back(mem_cfg_t(info->mem0_base, info->mem0_size)); + for (int i = 0; i < info->nharts; i++) + hartids.push_back(i); + + cfg = new cfg_t(std::make_pair(0, 0), + nullptr, + info->isa.c_str(), + "MSU", + "vlen:128,elen:64", + false, + endianness_little, + info->pmpregions, + mem_cfg, + hartids, + false, + 0 + ); + + std::vector> mems = make_mems(cfg->mem_layout()); + rom_device_t *boot_rom = new rom_device_t(info->bootrom); + mem_t *boot_addr_reg = new mem_t(0x1000); + uint64_t default_boot_addr = 0x80000000; + std::vector> plugin_devices; + boot_addr_reg->store(0, 8, (const uint8_t*)(&default_boot_addr)); + plugin_devices.push_back(std::pair(0x10000, boot_rom)); + plugin_devices.push_back(std::pair(0x4000, boot_addr_reg)); + + s_vpi_vlog_info vinfo; + if (!vpi_get_vlog_info(&vinfo)) + abort(); + std::vector htif_args; + bool in_permissive = false; + bool cospike_debug = false; + for (int i = 1; i < vinfo.argc; i++) { + std::string arg(vinfo.argv[i]); + if (arg == "+permissive") { + in_permissive = true; + } else if (arg == "+permissive-off") { + in_permissive = false; + } else if (arg == "+cospike_debug") { + cospike_debug = true; + } else if (!in_permissive) { + htif_args.push_back(arg); + } + } + + debug_module_config_t dm_config = { + .progbufsize = 2, + .max_sba_data_width = 0, + .require_authentication = false, + .abstract_rti = 0, + .support_hasel = true, + .support_abstract_csr_access = true, + .support_abstract_fpr_access = true, + .support_haltgroups = true, + .support_impebreak = true + }; + + printf("%s\n", info->isa.c_str()); + for (int i = 0; i < htif_args.size(); i++) { + printf("%s\n", htif_args[i].c_str()); + } + + sim = new sim_t(cfg, false, + mems, + plugin_devices, + htif_args, + dm_config, + nullptr, + false, + nullptr, + false, + nullptr + ); + + sim->configure_log(true, true); + // Use our own reset vector + for (int i = 0; i < info->nharts; i++) { + sim->get_core(hartid)->get_state()->pc = 0x10040; + } + sim->set_debug(cospike_debug); + printf("Setting up htif for spike cosim\n"); + ((htif_t*)sim)->start(); + printf("Spike cosim started\n"); + tohost_addr = ((htif_t*)sim)->get_tohost_addr(); + fromhost_addr = ((htif_t*)sim)->get_fromhost_addr(); + printf("Tohost : %lx\n", tohost_addr); + printf("Fromhost: %lx\n", fromhost_addr); + } + + processor_t* p = sim->get_core(hartid); + state_t* s = p->get_state(); + uint64_t s_pc = s->pc; + if (interrupt) { + printf("%d interrupt %lx\n", cycle, cause); + uint64_t interrupt_cause = cause & 0x7FFFFFFFFFFFFFFF; + if (interrupt_cause == 3) { + s->mip->backdoor_write_with_mask(MIP_MSIP, MIP_MSIP); + } else { + printf("Unknown interrupt %lx\n", interrupt_cause); + } + } + if (exception) + printf("%d exception %lx\n", cycle, cause); + if (valid) { + printf("%d Cosim: %lx", cycle, iaddr); + if (has_wdata) { + printf(" %lx", wdata); + } + printf("\n"); + } + if (valid || interrupt || exception) + p->step(1); + + if (valid) { + if (s_pc != iaddr) { + printf("%d PC mismatch %lx != %lx\n", cycle, s_pc, iaddr); + exit(1); + } + + // Try to remember magic_mem addrs, and ignore these in the future + auto& mem_write = s->log_mem_write; + if (!mem_write.empty() && tohost_addr && std::get<0>(mem_write[0]) == tohost_addr) { + reg_t wdata = std::get<1>(mem_write[0]); + if (wdata >= info->mem0_base && wdata < (info->mem0_base + info->mem0_size)) { + printf("Probable magic mem %x\n", wdata); + magic_addrs.insert(wdata); + } + } + + if (has_wdata) { + auto& log = s->log_reg_write; + auto& mem_read = s->log_mem_read; + + for (auto regwrite : log) { + int rd = regwrite.first >> 4; + int type = regwrite.first & 0xf; + // 0 => int + // 1 => fp + // 2 => vec + // 3 => vec hint + // 4 => csr + if ((rd != 0 && type == 0) || type == 1) { + // Override reads from some CSRs + if ((insn & 0xfff0007f) == 0xf1300073 || // mimpid + (insn & 0xfff0007f) == 0xf1200073 || // marchid + (insn & 0xfff0007f) == 0xf1100073 || // mvendorid + (insn & 0xfff0007f) == 0xb0000073 || // mcycle + (insn & 0xfff0007f) == 0xb0200073 // minstret + ) { + printf("CSR override\n"); + s->XPR.write(rd, wdata); + } else if (!mem_read.empty() && + ((magic_addrs.count(std::get<0>(mem_read[0])) || + tohost_addr && std::get<0>(mem_read[0]) == tohost_addr) || + (fromhost_addr && std::get<0>(mem_read[0]) == fromhost_addr))) { + // Don't check reads from tohost, or reads from magic memory + // Technically this could be buggy because log_mem_read only reports vaddrs, but + // no software ever should access tohost/fromhost with vaddrs anyways + printf("To/From host read override\n"); + s->XPR.write(rd, wdata); + } else if (wdata != regwrite.second.v[0]) { + printf("%d wdata mismatch reg %d %lx != %lx\n", cycle, rd, regwrite.second.v[0], wdata); + exit(1); + } + } + } + } + } +} +// } diff --git a/generators/chipyard/src/main/resources/vsrc/cospike.v b/generators/chipyard/src/main/resources/vsrc/cospike.v new file mode 100644 index 00000000..26050675 --- /dev/null +++ b/generators/chipyard/src/main/resources/vsrc/cospike.v @@ -0,0 +1,78 @@ +import "DPI-C" function void cospike_set_sysinfo( + input string isa, + input int pmpregions, + input longint mem0_base, + input longint mem0_size, + input int nharts, + input string bootrom + ); + +import "DPI-C" function void cospike_cosim(input longint cycle, + input longint hartid, + input bit has_wdata, + input bit valid, + input longint iaddr, + input int insn, + input bit exception, + input bit interrupt, + input longint cause, + input longint wdata + ); + + +module CospikeResources #( + parameter ISA, + parameter PMPREGIONS, + parameter MEM0_BASE, + parameter MEM0_SIZE, + parameter NHARTS, + parameter BOOTROM) + (); + initial begin + cospike_set_sysinfo(ISA, PMPREGIONS, MEM0_BASE, MEM0_SIZE, NHARTS, BOOTROM); + end; +endmodule; // CospikeResources + + +module SpikeCosim ( + input clock, + input reset, + + input [63:0] cycle, + + input [63:0] hartid, + + input trace_0_valid, + input [63:0] trace_0_iaddr, + input [31:0] trace_0_insn, + input trace_0_exception, + input trace_0_interrupt, + input [63:0] trace_0_cause, + input trace_0_has_wdata, + input [63:0] trace_0_wdata, + + input trace_1_valid, + input [63:0] trace_1_iaddr, + input [31:0] trace_1_insn, + input trace_1_exception, + input trace_1_interrupt, + input [63:0] trace_1_cause, + input trace_1_has_wdata, + input [63:0] trace_1_wdata + ); + + always @(posedge clock) begin + if (!reset) begin + if (trace_0_valid || trace_0_exception || trace_0_cause) begin + cospike_cosim(cycle, hartid, trace_0_has_wdata, trace_0_valid, trace_0_iaddr, + trace_0_insn, trace_0_exception, trace_0_interrupt, trace_0_cause, + trace_0_wdata); + end + if (trace_1_valid || trace_1_exception || trace_1_cause) begin + cospike_cosim(cycle, hartid, trace_1_has_wdata, trace_1_valid, trace_1_iaddr, + trace_1_insn, trace_1_exception, trace_1_interrupt, trace_1_cause, + trace_1_wdata); + end + end + end +endmodule; // CospikeCosim diff --git a/generators/chipyard/src/main/scala/Cospike.scala b/generators/chipyard/src/main/scala/Cospike.scala new file mode 100644 index 00000000..1ebf6253 --- /dev/null +++ b/generators/chipyard/src/main/scala/Cospike.scala @@ -0,0 +1,104 @@ +package chipyard + +import chisel3._ +import chisel3.experimental.{IntParam, StringParam, IO} +import chisel3.util._ + +import freechips.rocketchip.config.{Parameters, Field, Config} +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.devices.tilelink._ +import freechips.rocketchip.util._ + +import testchipip.TileTraceIO + +class CospikeResources( + isa: String, + pmpregions: Int, + mem0_base: BigInt, + mem0_size: BigInt, + nharts: Int, + bootrom: String +) extends BlackBox(Map( + "ISA" -> StringParam(isa), + "PMPREGIONS" -> IntParam(pmpregions), + "MEM0_BASE" -> IntParam(mem0_base), + "MEM0_SIZE" -> IntParam(mem0_size), + "NHARTS" -> IntParam(nharts), + "BOOTROM" -> StringParam(bootrom) +)) with HasBlackBoxResource { + val io = IO(new Bundle {}) + addResource("/csrc/cospike.cc") + addResource("/vsrc/cospike.v") +} + +trait CanHaveSpikeCosim { this: ChipyardSystem => + InModuleBody { + val isa = tiles.headOption.map(_.isaDTS).getOrElse("") + val mem0_base = p(ExtMem).map(_.master.base).getOrElse(BigInt(0)) + val mem0_size = p(ExtMem).map(_.master.size).getOrElse(BigInt(0)) + val pmpregions = tiles.headOption.map(_.tileParams.core.nPMPs).getOrElse(0) + val nharts = tiles.size + val bootrom = bootROM.map(_.module.contents.toArray.mkString(" ")).getOrElse("") + val resources = Module(new CospikeResources(isa, pmpregions, mem0_base, mem0_size, nharts, bootrom)) + } +} + +class SpikeCosim extends BlackBox with HasBlackBoxResource +{ + addResource("/csrc/cospike.cc") + addResource("/vsrc/cospike.v") + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val cycle = Input(UInt(64.W)) + val hartid = Input(UInt(64.W)) + val trace = Input(Vec(2, new Bundle { + val valid = Bool() + val iaddr = UInt(64.W) + val insn = UInt(32.W) + val exception = Bool() + val interrupt = Bool() + val cause = UInt(64.W) + val has_wdata = Bool() + val wdata = UInt(64.W) + })) + }) +} + +object SpikeCosim +{ + def apply(trace: TileTraceIO, hartid: Int) = { + val cosim = Module(new SpikeCosim) + val cycle = withClockAndReset(trace.clock, trace.reset) { + val r = RegInit(0.U(64.W)) + r := r + 1.U + r + } + cosim.io.clock := trace.clock + cosim.io.reset := trace.reset + require(trace.numInsns <= 2) + cosim.io.cycle := cycle + cosim.io.trace.map(t => { + t.valid := false.B + t.iaddr := 0.U + t.insn := 0.U + t.exception := false.B + t.interrupt := false.B + t.cause := 0.U + }) + cosim.io.hartid := hartid.U + for (i <- 0 until trace.numInsns) { + cosim.io.trace(i).valid := trace.insns(i).valid + val signed = Wire(SInt(64.W)) + signed := trace.insns(i).iaddr.asSInt + cosim.io.trace(i).iaddr := signed.asUInt + cosim.io.trace(i).insn := trace.insns(i).insn + cosim.io.trace(i).exception := trace.insns(i).exception + cosim.io.trace(i).interrupt := trace.insns(i).interrupt + cosim.io.trace(i).cause := trace.insns(i).cause + cosim.io.trace(i).has_wdata := trace.insns(i).wdata.isDefined.B + cosim.io.trace(i).wdata := trace.insns(i).wdata.getOrElse(0.U) + } + } +} diff --git a/generators/chipyard/src/main/scala/DigitalTop.scala b/generators/chipyard/src/main/scala/DigitalTop.scala index 2777ba36..dc01d200 100644 --- a/generators/chipyard/src/main/scala/DigitalTop.scala +++ b/generators/chipyard/src/main/scala/DigitalTop.scala @@ -34,6 +34,7 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem with chipyard.clocking.HasChipyardPRCI // Use Chipyard reset/clock distribution with fftgenerator.CanHavePeripheryFFT // Enables optionally having an MMIO-based FFT block with constellation.soc.CanHaveGlobalNoC // Support instantiating a global NoC interconnect + with chipyard.CanHaveSpikeCosim // Support instantiating spike-based co-simulation { override lazy val module = new DigitalTopModule(this) } diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala index 5ff78a65..627644c3 100644 --- a/generators/chipyard/src/main/scala/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/HarnessBinders.scala @@ -21,7 +21,7 @@ import barstools.iocell.chisel._ import testchipip._ -import chipyard.{HasHarnessSignalReferences, HarnessClockInstantiatorKey} +import chipyard._ import chipyard.clocking.{HasChipyardPRCI} import chipyard.iobinders.{GetSystemParameters, JTAGChipIO, ClockWithFreq} @@ -333,6 +333,13 @@ class WithSimDromajoBridge extends ComposeHarnessBinder({ } }) +class WithCospikeBridge extends ComposeHarnessBinder({ + (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[TraceOutputTop]) => { + ports.map { p => p.traces.zipWithIndex.map(t => SpikeCosim(t._1, t._2)) } + } +}) + + class WithCustomBootPinPlusArg extends OverrideHarnessBinder({ (system: CanHavePeripheryCustomBootPin, th: HasHarnessSignalReferences, ports: Seq[Bool]) => { val pin = PlusArg("custom_boot_pin", width=1) diff --git a/generators/chipyard/src/main/scala/config/BoomConfigs.scala b/generators/chipyard/src/main/scala/config/BoomConfigs.scala index bbc816f7..e212fd1a 100644 --- a/generators/chipyard/src/main/scala/config/BoomConfigs.scala +++ b/generators/chipyard/src/main/scala/config/BoomConfigs.scala @@ -54,3 +54,9 @@ class DromajoBoomConfig extends Config( new boom.common.WithNSmallBooms(1) ++ new chipyard.config.WithSystemBusWidth(128) ++ new chipyard.config.AbstractConfig) + +class MediumBoomCosimConfig extends Config( + new chipyard.harness.WithCospikeBridge ++ // attach spike-cosim + new chipyard.config.WithTraceIO ++ // enable the traceio + new boom.common.WithNMediumBooms(1) ++ + new chipyard.config.AbstractConfig)