added driver sim

driver/sw/Makefile

@@ -0,0 +1,71 @@
+
+DRV_CFLAGS += -O0 -g -Wall -Wextra -pedantic -Wfatal-errors
+
+DRV_CFLAGS += -I/tools/opae/1.4.0/include
+
+DRV_LDFLAGS += -L/tools/opae/1.4.0/lib
+
+# stack execution protection
+DRV_LDFLAGS +=-z noexecstack
+
+# data relocation and projection
+DRV_LDFLAGS +=-z relro -z now
+
+# stack buffer overrun detection
+# Note that CentOS 7 has gcc 4.8 by default.  When we switch
+# to a system with gcc 4.9 or newer this should be changed to
+# CFLAGS="-fstack-protector-strong"
+DRV_CFLAGS +=-fstack-protector
+
+# Position independent code
+DRV_CFLAGS += -fPIC
+
+DRV_LDFLAGS += -luuid
+
+DRV_LDFLAGS += -shared
+
+FPGA_LIBS += -lopae-c
+ASE_LIBS += -lopae-c-ase
+
+CXXFLAGS += -std=c++17 -O0 -g -Wall -Wextra -pedantic -Wfatal-errors
+
+LDFLAGS += -L.
+
+PROJECT = libvxdrv.so
+PROJECT_ASE = libvxdrv_ase.so
+
+AFU_JSON_INFO = vortex_afu.h
+
+all: $(PROJECT) $(PROJECT_ASE) test test_ase
+
+# AFU info from JSON file, including AFU UUID
+$(AFU_JSON_INFO): ../hw/vortex_afu.json
+	afu_json_mgr json-info --afu-json=$^ --c-hdr=$@
+
+$(PROJECT): vx_driver.o
+	$(CC) $(DRV_CFLAGS) $^ $(DRV_LDFLAGS) $(FPGA_LIBS) -o $@
+
+$(PROJECT_ASE): vx_driver.o
+	$(CC) $(DRV_CFLAGS) -DUSE_ASE $^ $(DRV_LDFLAGS) $(ASE_LIBS) -o $@
+
+test: test.o $(PROJECT)
+	$(CXX) $(CXXFLAGS) test.o $(LDFLAGS) -lvxdrv -o $@
+
+test_ase: test.o $(PROJECT_ASE)
+	$(CXX) $(CXXFLAGS) -DUSE_ASE test.o $(LDFLAGS) -lvxdrv_ase -o $@
+
+vx_driver.o: vx_driver.c 
+	$(CC) $(DRV_CFLAGS) -c $^ -o $@
+
+test.o: test.cpp $(AFU_JSON_INFO) 
+	$(CXX) $(CXXFLAGS) -c test.cpp -o $@
+
+.depend: vx_driver.c test.cpp
+	$(CXX) $(CXXFLAGS) -MM $^ > .depend;
+
+clean:
+	rm -rf $(PROJECT) $(PROJECT_ASE) test test_ase $(AFU_JSON_INFO) *.so *.o .depend 
+
+ifneq ($(MAKECMDGOALS),clean)
+    -include .depend
+endif

driver/sw/test.cpp

@@ -0,0 +1,142 @@
+#include <vx_driver.h>
+#include <iostream>
+#include <fstream>
+#include <unistd.h>
+
+#include "utils.h"
+
+#define CACHE_LINESIZE		64
+
+const char* program_file = nullptr;
+
+static void show_usage() {
+   std::cout << "Vortex Driver Test." << std::endl;
+   std::cout << "Usage: [-f: program] [-h: help]" << std::endl;
+}
+
+static void parse_args(int argc, char **argv) {
+  int c;
+  while ((c = getopt(argc, argv, "f:h?")) != -1) {
+    switch (c) {
+    case 'f': {
+      program_file = optarg;
+    } break;
+    case 'h':
+    case '?': {
+      show_usage();
+      exit(0);
+    } break;
+    default:
+      show_usage();
+      exit(-1);
+    }
+  }
+
+  if (nullptr == program_file) {
+    show_usage();
+    exit(-1);
+  }
+}
+
+static int upload_program(vx_device_h device, const char* filename, uint32_t transfer_size = 16 * VX_CACHE_LINESIZE) {
+  std::ifstream ifs(filename);
+  if (!ifs) {
+    std::cout << "error: " << filename << " not found" << std::endl;
+    return -1;
+  }
+
+  // allocate device buffer
+  auto buffer = vx_buf_alloc(device, transfer_size);
+  if (nullptr == buffer)
+    return -1; 
+
+  // get buffer address
+  auto buf_ptr = (uint8_t*)vs_buf_ptr(buffer);
+  
+  //
+  // copy hex program
+  //
+
+  char line[ihex_t::MAX_LINE_SIZE];
+  uint32_t hex_offset = 0;
+  uint32_t prev_hex_address = 0;
+  uint32_t dest_address = -1;
+  uint32_t src_offset = 0;
+
+  while (true) {
+    ifs.getline(line, ihex_t::MAX_LINE_SIZE);
+    if (!ifs) 
+      break;
+
+    ihex_t ihex;
+    parse_ihex_line(line, &ihex);
+    if (ihex.is_eof)
+      break;
+
+    if (ihex.has_offset) {
+      hex_offset = ihex.offset;
+    }
+
+    if (ihex.data_size != 0) {
+        auto hex_address = ihex.address + hex_offset;
+        if (dest_address == (uint32_t)-1) {
+          dest_address = (hex_address / VX_CACHE_LINESIZE) * VX_CACHE_LINESIZE;          
+          src_offset = hex_address - dest_address;
+        } else {
+          auto delta = hex_address - prev_hex_address;
+          src_offset += delta;
+        }
+        for (uint32_t i = 0; i < ihex.data_size; ++i) {          
+          if (src_offset >= transfer_size) {
+            // flush current batch to FPGA
+            vx_copy_to_fpga(buffer, dest_address, transfer_size, 0);
+            dest_address = (hex_address/ VX_CACHE_LINESIZE) * VX_CACHE_LINESIZE;
+            src_offset = hex_address - dest_address;            
+          }
+          buf_ptr[src_offset++] = ihex.data[i];
+          ++hex_address;
+        }
+        prev_hex_address = hex_address;
+    }
+  }
+
+  // flush last batch to FPGA
+  if (src_offset) {
+    vx_copy_to_fpga(buffer, dest_address, src_offset, 0);
+  }
+
+  vx_buf_release(buffer);
+
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  // parse command arguments
+  parse_args(argc, argv);
+
+  // open device connection
+  auto device = vx_dev_open();
+
+  // upload program
+  if (0 != upload_program(device, program_file)) {
+    vx_dev_close(device);
+    return -1;  
+  }
+
+  // start device
+  if (0 != vx_start(device)) {
+    vx_dev_close(device);
+    return -1;  
+  }
+
+  // wait for completion
+  if (0 != vx_ready_wait(device, -1)) {
+    vx_dev_close(device);
+    return -1;  
+  }
+  
+  // close device
+  vx_dev_close(device);
+
+  return 0;
+}

driver/sw/utils.cpp

@@ -0,0 +1,71 @@
+
+
+#include <iostream>
+#include "utils.h"
+
+static uint32_t hti_old(char c) {
+      if (c >= 'A' && c <= 'F')
+          return c - 'A' + 10;
+      if (c >= 'a' && c <= 'f')
+          return c - 'a' + 10;
+      return c - '0';
+  }
+
+static uint32_t hToI_old(char *c, uint32_t size) {
+    uint32_t value = 0;
+    for (uint32_t i = 0; i < size; i++) {
+        value += hti_old(c[i]) << ((size - i - 1) * 4);
+    }
+    return value;
+}
+
+
+int parse_ihex_line(char* line, ihex_t* out) {
+    if (line[0] != ':') {
+      std::cout << "error: invalid line entry!" << std::endl;  
+      return -1;
+    }
+    
+    uint32_t data_size = 0;
+    uint32_t address = 0;    
+    uint32_t offset = 0;
+    bool has_offset = false;
+    bool is_eof = false;
+
+    auto record_type = hToI_old(line + 7, 2);
+
+    switch (record_type) {
+    case 0: { // data  
+      data_size = hToI_old(line + 1, 2);
+      address = hToI_old(line + 3, 4);
+      for (uint32_t i = 0; i < data_size; i++) {
+          out->data[i] = hToI_old(line + 9 + i * 2, 2); 
+      }
+    } break;
+    case 1: // end of file
+      is_eof = true;
+      break;
+    case 2: // extended segment address
+      offset = hToI_old(line + 9, 4) << 4;
+      has_offset = true;
+      break;
+    case 3: // start segment address
+      break;
+    case 4: // extended linear address
+      offset = hToI_old(line + 9, 4) << 16;
+      has_offset = true;
+      break;
+    case 5: // start linear address
+      break;
+    default:
+      return -1;
+    }
+
+    out->address = address;
+    out->data_size = data_size;
+    out->offset = offset;
+    out->has_offset = has_offset;
+    out->is_eof = is_eof;
+
+    return 0;
+  }

driver/sw/utils.h

@@ -0,0 +1,15 @@
+
+#pragma once  
+
+struct ihex_t {
+  static constexpr int MAX_LINE_SIZE = 524;
+  static constexpr int MAX_DATA_SIZE = 255;
+  uint8_t data[MAX_DATA_SIZE];
+  uint32_t address;    
+  uint32_t data_size;    
+  uint32_t offset;
+  bool has_offset;
+  bool is_eof;
+};
+
+int parse_ihex_line(char* line, ihex_t* out);

driver/sw/vx_driver.c

@@ -0,0 +1,259 @@
+#include "vx_driver.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <uuid/uuid.h>
+
+#include <opae/fpga.h>
+
+// MMIO Address Mappings
+#define AFU_ID                  AFU_ACCEL_UUID
+
+#define MMIO_COPY_IO_ADDRESS    0X120
+#define MMIO_COPY_AVM_ADDRESS   0x100
+#define MMIO_COPY_DATA_SIZE     0X118
+
+#define MMIO_CMD_TYPE           0X110		// MMIO location set by SW to denote read/write. read: 3; write: 1; vortex: 7
+#define MMIO_READY_FOR_CMD      0X198
+
+#define CHECK_RES(_expr)                                            \
+   do {                                                             \
+     fpga_result res = _expr;                                       \
+     if (res == FPGA_OK)                                            \
+       break;                                                       \
+     printf("OPAE Error: '%s' returned %d!\n", #_expr, (int)res);   \
+     return -1;                                                     \
+   } while (false)
+
+///////////////////////////////////////////////////////////////////////////////
+
+typedef struct vx_buffer_ {
+    uint64_t wsid;
+    volatile void* host_ptr;
+    uint64_t io_addr;
+    fpga_handle hdevice;
+    size_t size;
+} vx_buffer_t;
+
+static size_t align_size(size_t size) {
+    return VX_CACHE_LINESIZE * ((size + VX_CACHE_LINESIZE - 1) / VX_CACHE_LINESIZE);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+// Search for an accelerator matching the requested UUID and connect to it
+// Convert this to void if required as storing the fpga_handle to params variable
+extern vx_device_h vx_dev_open(const char *accel_uuid) {
+    fpga_properties filter = NULL;
+    fpga_result res;
+    fpga_guid guid;
+    fpga_token accel_token;
+    uint32_t num_matches;
+    fpga_handle accel_handle;
+
+    // Set up a filter that will search for an accelerator
+    fpgaGetProperties(NULL, &filter);
+    fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);
+
+    // Add the desired UUID to the filter
+    uuid_parse(accel_uuid, guid);
+    fpgaPropertiesSetGUID(filter, guid);
+
+    // Do the search across the available FPGA contexts
+    num_matches = 1;
+    fpgaEnumerate(&filter, 1, &accel_token, 1, &num_matches);
+
+    // Not needed anymore
+    fpgaDestroyProperties(&filter);
+
+    if (num_matches < 1) {
+        fprintf(stderr, "Accelerator %s not found!\n", accel_uuid);
+        return NULL;
+    }
+
+    // Open accelerator
+    res = fpgaOpen(accel_token, &accel_handle, 0);
+    if (FPGA_OK != res) {
+        return NULL;
+    }
+
+    // Done with token
+    fpgaDestroyToken(&accel_token);
+
+    return accel_handle;
+}
+
+// Close the fpga when all the operations are done
+extern int vx_dev_close(vx_device_h hdevice) {
+    if (NULL == hdevice)
+        return -1;
+
+    fpgaClose(hdevice);
+
+    return 0;
+}
+
+extern vx_buffer_h vx_buf_alloc(vx_device_h hdevice, size_t size) {
+    fpga_result res;
+    void* host_ptr;
+    uint64_t wsid;
+    uint64_t io_addr;
+    vx_buffer_t* buffer;
+
+    if (NULL == hdevice)
+        return NULL;
+
+    size_t asize = align_size(size);
+
+    res = fpgaPrepareBuffer(hdevice, asize, &host_ptr, &wsid, 0);
+    if (FPGA_OK != res) {
+        return NULL;
+    }
+
+    // Get the physical address of the buffer in the accelerator
+    res = fpgaGetIOAddress(hdevice, wsid, &io_addr);
+    if (FPGA_OK != res) {
+        fpgaReleaseBuffer(hdevice, wsid);
+        return NULL;
+    }
+
+    buffer = (vx_buffer_t*)malloc(sizeof(vx_buffer_t));
+    if (NULL == buffer) {
+        fpgaReleaseBuffer(hdevice, wsid);
+        return NULL;
+    }
+
+    buffer->wsid = wsid;
+    buffer->host_ptr = host_ptr;
+    buffer->io_addr = io_addr;
+    buffer->hdevice = hdevice;
+    buffer->size = size;
+
+    return (vx_buffer_h)buffer;
+}
+
+extern volatile void* vs_buf_ptr(vx_buffer_h hbuffer) {
+    vx_buffer_t* buffer = ((vx_buffer_t*)hbuffer);
+    if (NULL == buffer)
+        return NULL;
+
+    return buffer->host_ptr;
+}
+
+extern int vx_buf_release(vx_buffer_h hbuffer) {
+    vx_buffer_t* buffer = ((vx_buffer_t*)hbuffer);
+    if (NULL == buffer)
+        return -1;
+
+    fpgaReleaseBuffer(buffer->hdevice, buffer->wsid);
+
+    free(hbuffer);
+
+    return 0;
+}
+
+// Check if HW is ready for SW
+static int ready_for_sw(fpga_handle hdevice) {
+    uint64_t data = 0;
+    struct timespec sleep_time; 
+    
+#ifdef USE_ASE
+    sleep_time.tv_sec = 1;
+    sleep_time.tv_nsec = 0;
+#else
+    sleep_time.tv_sec = 0;
+    sleep_time.tv_nsec = 1000000;
+#endif
+
+    do {
+        CHECK_RES(fpgaReadMMIO64(hdevice, 0, MMIO_READY_FOR_CMD, &data));
+        nanosleep(&sleep_time, NULL);
+    } while (data != 0x1);
+
+    return 0;
+}
+
+extern int vx_copy_to_fpga(vx_buffer_h hbuffer, size_t dest_addr, size_t size, size_t src_offset) {
+    vx_buffer_t* buffer = ((vx_buffer_t*)hbuffer);
+
+    // bound checking
+    if (size + src_offset > buffer->size)
+        return -1;
+
+    // Ensure ready for new command
+    if (ready_for_sw(buffer->hdevice) != 0)
+        return -1;
+
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_COPY_AVM_ADDRESS, dest_addr));
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_COPY_IO_ADDRESS, (buffer->io_addr + src_offset)/VX_CACHE_LINESIZE));
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_COPY_DATA_SIZE, size));   
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_CMD_TYPE, 1)); // WRITE CMD
+
+    // Wait for the write operation to finish
+    return ready_for_sw(buffer->hdevice);
+}
+
+extern int vx_copy_from_fpga(vx_buffer_h hbuffer, size_t src_addr, size_t size, size_t dest_offset) {
+    vx_buffer_t* buffer = ((vx_buffer_t*)hbuffer);
+
+    // bound checking
+    if (size + dest_offset > buffer->size)
+        return -1;
+
+    // Ensure ready for new command
+    if (ready_for_sw(buffer->hdevice) != 0)
+        return -1;
+
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_COPY_AVM_ADDRESS, src_addr));
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_COPY_IO_ADDRESS, (buffer->io_addr + dest_offset)/VX_CACHE_LINESIZE));
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_COPY_DATA_SIZE, size));   
+    CHECK_RES(fpgaWriteMMIO64(buffer->hdevice, 0, MMIO_CMD_TYPE, 3)); // READ CMD
+
+    // Wait for the write operation to finish
+    return ready_for_sw(buffer->hdevice);
+}
+
+extern int vx_start(vx_device_h hdevice) {
+    if (NULL == hdevice)
+        return -1;
+
+    // Ensure ready for new command
+    if (ready_for_sw(hdevice) != 0)
+        return -1;
+
+    CHECK_RES(fpgaWriteMMIO64(hdevice, 0, MMIO_CMD_TYPE, 7)); // START CMD    
+
+    return 0;
+}
+
+extern int vx_ready_wait(vx_device_h hdevice, long long timeout) {
+    if (NULL == hdevice)
+        return -1;
+
+    uint64_t data = 0;
+    struct timespec sleep_time; 
+
+#ifdef USE_ASE
+    sleep_time.tv_sec = 1;
+    sleep_time.tv_nsec = 0;
+#else
+    sleep_time.tv_sec = 0;
+    sleep_time.tv_nsec = 1000000;
+#endif
+
+    // to milliseconds
+    long long sleep_time_ms = (sleep_time.tv_sec * 1000) + (sleep_time.tv_nsec / 1000000);
+    
+    do {
+        CHECK_RES(fpgaReadMMIO64(hdevice, 0, MMIO_READY_FOR_CMD, &data));
+        nanosleep(&sleep_time, NULL);
+        sleep_time_ms -= sleep_time_ms;
+        if (timeout <= sleep_time_ms)
+            break;        
+    } while (data != 0x1);
+
+    return 0;
+}

driver/sw/vx_driver.h

@@ -0,0 +1,47 @@
+#ifndef __VX_DRIVER_H__
+#define __VX_DRIVER_H__
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* vx_device_h;
+
+typedef void* vx_buffer_h;
+
+#define VX_CACHE_LINESIZE 64
+
+// open the device and connect to it
+vx_device_h vx_dev_open();
+
+// Close the device when all the operations are done
+int vx_dev_close(vx_device_h hdevice);
+
+// Allocate shared buffer with device
+vx_buffer_h vx_buf_alloc(vx_device_h hdevice, size_t size);
+
+// Get host pointer address  
+void* vs_buf_ptr(vx_buffer_h hbuffer);
+
+// release buffer
+int vx_buf_release(vx_buffer_h hbuffer);
+
+// Copy bytes from buffer to device local memory
+int vx_copy_to_fpga(vx_buffer_h hbuffer, size_t dest_addr, size_t size, size_t src_offset);
+
+// Copy bytes from device local memory to buffer
+int vx_copy_from_fpga(vx_buffer_h hbuffer, size_t src_addr, size_t size, size_t dst_offset);
+
+// Start device execution
+int vx_start(vx_device_h hdevice);
+
+// Wait for device ready with milliseconds timeout
+int vx_ready_wait(vx_device_h hdevice, long long timeout);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // __VX_DRIVER_H__