cachelab started

cachelab/cbsl/tests/CMakeLists.txt

@@ -0,0 +1,106 @@
+#
+# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+link_libraries(${CBSL_LIB} ${ZSTD_LIB})
+
+add_executable(scalar_data scalar_data.c)
+add_executable(single_array single_array.c)
+add_executable(multi_array multi_array.c)
+add_executable(utility utility.c)
+add_executable(split_compression split_compression.c)
+add_executable(variable_size_array variable_size_array.c)
+add_executable(various_size_array various_size_array.c)
+
+add_executable(cbsl_record       cbsl_record.c)
+add_executable(cbsl_record_array cbsl_record_array.c)
+
+add_executable(fortran_bindings fortran_bindings.f90)
+target_link_libraries(fortran_bindings ${CBSL_FLIB} ${CBSL_LIB} ${ZSTD_LIB})
+
+add_test(NAME utility COMMAND utility)
+
+
+#
+# very short data compression/decompression
+#
+add_test(NAME scalar_data_compression   COMMAND scalar_data -c)
+add_test(NAME scalar_data_decompression COMMAND scalar_data -d)
+set_tests_properties(scalar_data_decompression PROPERTIES DEPENDS scalar_data_compression)
+
+
+#
+# boundary checking with single array
+#
+function(add_simple_test_size_boundary DATA_SIZE)
+  math(EXPR DATA_SIZE_B "${DATA_SIZE} - 1")
+  add_test(NAME single_array_${DATA_SIZE_B} COMMAND single_array ${DATA_SIZE_B})
+  add_test(NAME single_array_${DATA_SIZE}   COMMAND single_array ${DATA_SIZE})
+  math(EXPR DATA_SIZE_B "${DATA_SIZE} + 1")
+  add_test(NAME single_array_${DATA_SIZE_B} COMMAND single_array ${DATA_SIZE_B})
+endfunction()
+
+foreach(count RANGE 10 20)
+  math(EXPR DATA_SIZE "1 << ${count}")
+  add_simple_test_size_boundary(${DATA_SIZE})
+endforeach()
+
+
+#
+# boundary checking with multiple arrays
+#
+function(add_multiple_test_size NUM_VARS)
+  foreach(count RANGE 10 20)
+    math(EXPR DATA_SIZE "1 << ${count}")
+    add_test(NAME multi_array_${DATA_SIZE}_by_${NUM_VARS}  COMMAND multi_array ${NUM_VARS} ${DATA_SIZE})
+  endforeach()
+endfunction()
+
+add_multiple_test_size(2)
+add_multiple_test_size(3)
+add_multiple_test_size(11)
+
+
+#
+# test compression/decompression form
+#
+add_test(NAME variable_size_array COMMAND variable_size_array)
+math(EXPR DATA_SIZE "1 << 17")
+add_test(NAME buffered_read_write COMMAND variable_size_array ${DATA_SIZE})
+math(EXPR DATA_SIZE "(1 << 17) + 1")
+add_test(NAME immediate_read_write COMMAND variable_size_array ${DATA_SIZE})
+
+add_test(NAME various_size_array COMMAND various_size_array)
+
+
+add_test(NAME cbsl_record        COMMAND cbsl_record)
+add_test(NAME cbsl_record_array  COMMAND cbsl_record_array)
+
+
+#
+# large array compression/decompression by splitting small block
+#
+math(EXPR BLOCK_SIZE "1 << 18") # 256 KiB
+math(EXPR TOTAL_SIZE "1 << 28") # 256 MiB
+add_test(NAME split_compression_256KiB_256MiB COMMAND split_compression ${BLOCK_SIZE} ${TOTAL_SIZE})
+
+math(EXPR BLOCK_SIZE "(1 << 18) - (1 << 16)") # 192 KiB
+math(EXPR TOTAL_SIZE "1 << 28")               # 256 MiB
+add_test(NAME split_compression_192KiB_256MiB COMMAND split_compression ${BLOCK_SIZE} ${TOTAL_SIZE})
+
+
+#
+# Fortran bindings API
+#
+add_test(NAME fortran_bindings COMMAND fortran_bindings)

cachelab/cbsl/tests/cbsl_record.c

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+#define CHECK_BINARY(X,Y)    {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
+
+static const int    data0 = 43;
+static const double data1 = 3.14159265;
+static const int    data2 = 14142;
+
+extern void record(cbsl_mode, int*, double*, int*);
+
+char cname[128];
+
+int main(int argc, char** argv)
+{
+  sprintf(cname, "check_record_scalar.zst");
+
+  int    c0 = data0;
+  double c1 = data1;
+  int    c2 = data2;
+  record(cbsl_store_mode, &c0, &c1, &c2);
+
+  c0 = c2 = 0;
+  c1 = 0;
+  record(cbsl_load_mode, &c0, &c1, &c2);
+
+  CHECK_BINARY(c0, data0);
+  CHECK_BINARY(c1, data1);
+  CHECK_BINARY(c2, data2);
+
+  return 0;
+}
+
+void record(cbsl_mode mode, int* c0, double* c1, int* c2)
+{
+  cbsl_ctx* ctx = cbsl_open(mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_record(ctx, c0, sizeof(int)));
+  CBSL_ERROR_CHECK(cbsl_record(ctx, c1, sizeof(double)));
+  CBSL_ERROR_CHECK(cbsl_record(ctx, c2, sizeof(int)));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}

cachelab/cbsl/tests/cbsl_record_array.c

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+#define CHECK(X)             {if (!(X)) { fprintf(stderr, "error: check %s\n", (#X)); exit(1); }}
+#define CHECK_BINARY(X,Y)    {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
+
+typedef unsigned char byte_t;
+
+extern void rand_byte_t(uint64_t data_size, byte_t* a);
+extern void record(cbsl_mode, uint64_t* size, void** data);
+
+char cname[128];
+
+int main(int argc, char** argv)
+{
+  sprintf(cname, "check_record_scalar.zst");
+
+  uint64_t size = 1024 * 1024 * sizeof(byte_t);
+  byte_t*  data = (byte_t*)(malloc(size));
+
+  rand_byte_t(size, data);
+  record(cbsl_store_mode, &size, (void**) &data);
+
+  uint64_t rsize = 0;
+  byte_t*  rdata = NULL;
+  record(cbsl_load_mode, &rsize, (void**) &rdata);
+
+  CHECK(size == rsize);
+  CHECK(rdata != NULL);
+  for (uint64_t i = 0; i < (size/sizeof(byte_t)); ++i)
+  {
+    if (data[i] != rdata[i])
+    {
+      fprintf(stderr, "1: mismatch!\n");
+      exit(1);
+    }
+    else
+    {
+      rdata[i] = 0;
+    }
+  }
+
+  CHECK(size == rsize);
+  CHECK(rdata != NULL);
+  record(cbsl_load_mode, &rsize, (void**) &rdata);
+
+  for (uint64_t i = 0; i < (size/sizeof(byte_t)); ++i)
+  {
+    if (data[i] != rdata[i])
+    {
+      fprintf(stderr, "2: mismatch!\n");
+      exit(1);
+    }
+  }
+
+  free(data);
+  free(rdata);
+
+  return 0;
+}
+
+void record(cbsl_mode mode, uint64_t* size, void** data)
+{
+  cbsl_ctx* ctx = cbsl_open(mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_record_heap(ctx, data, size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void rand_byte_t(uint64_t data_size, byte_t* a)
+{
+  for(uint64_t i = 0; i < data_size; ++i)
+    a[i] = rand() % 255;
+}

cachelab/cbsl/tests/fortran_bindings.f90

@@ -0,0 +1,159 @@
+!
+! Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+!
+! Licensed under the Apache License, Version 2.0 (the "License");
+! you may not use this file except in compliance with the License.
+! You may obtain a copy of the License at
+!
+!     http://www.apache.org/licenses/LICENSE-2.0
+!
+! Unless required by applicable law or agreed to in writing, software
+! distributed under the License is distributed on an "AS IS" BASIS,
+! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+! See the License for the specific language governing permissions and
+! limitations under the License.
+!
+program test_fortran_bindings
+  use cbslf
+  integer, parameter :: array_size = 100000
+  real :: source_stack_array(array_size), dest_stack_array(array_size)
+  real, allocatable :: source_heap_array(:), dest_heap_array(:), dest_heap_array2(:)
+  character(128) :: source_string, dest_string
+  character(*), parameter :: cname = 'data_fortran.zst'
+
+  allocate(source_heap_array(array_size))
+  allocate(dest_heap_array(array_size))
+  call random_number(source_stack_array)
+  call random_number(source_heap_array)
+
+  source_string = 'abcdefgABCDEFG%'
+  dest_string   = ''
+
+  print *, 'serialize...'
+  call serialize
+
+  print *, 'deserialize...'
+  call deserialize
+
+  print *, 'compare...'
+  if (.not. compare_real_array(array_size, source_stack_array, dest_stack_array)) then
+    print *, 'fail: compare stack array'
+    stop 1
+  end if
+
+  if (.not. compare_real_array(array_size, source_heap_array, dest_heap_array)) then
+    print *, 'fail: compare heap array'
+    stop 1
+  end if
+
+  if (.not. compare_real_array(array_size, source_heap_array, dest_heap_array2)) then
+    print *, 'fail: compare heap array'
+    stop 1
+  end if
+
+  if (trim(source_string) /= trim(dest_string)) then
+    print *, 'fail: compare string'
+    stop 1
+  end if
+
+contains
+  subroutine serialize
+    implicit none
+    type(cbslf_context) :: ctx
+    integer(4) :: errcode
+
+    ctx = cbslf_open(cbslf_store_mode, cname, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_open(store)'
+      stop 1
+    end if
+
+    call cbslf_set_compression_level(ctx, 10, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_set_compression_level'
+      stop 1
+    end if
+
+    call cbslf_write(ctx, source_stack_array, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_write(stack array)'
+      stop 1
+    end if
+
+    call cbslf_write(ctx, source_heap_array, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_write(heap array)'
+      stop 1
+    end if
+
+    call cbslf_write(ctx, source_heap_array, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_write(heap array2)'
+      stop 1
+    end if
+
+    call cbslf_write(ctx, source_string, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_write(string)'
+      stop 1
+    end if
+
+    call cbslf_close(ctx)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_close'
+      stop 1
+    end if
+  end subroutine
+
+  subroutine deserialize
+    implicit none
+    type(cbslf_context) :: ctx
+    integer(4) :: errcode
+    ctx = cbslf_open(cbslf_load_mode, cname, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_open(load)'
+      stop 1
+    end if
+
+    call cbslf_read(ctx, dest_stack_array, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_read(stack array)'
+      stop 1
+    end if
+
+    call cbslf_read(ctx, dest_heap_array, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_read(heap array)'
+      stop 1
+    end if
+
+    call cbslf_record_heap(ctx, dest_heap_array2, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_record_heap(heap array2)'
+      stop 1
+    end if
+
+    call cbslf_read(ctx, dest_string, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_read(string)'
+      stop 1
+    end if
+
+    call cbslf_close(ctx, errcode)
+    if (errcode /= cbslf_success) then
+      print *, 'fail: cbslf_close'
+      stop 1
+    end if
+  end subroutine
+
+  function compare_real_array(n, a, b) result(ret)
+    implicit none
+    integer, intent(in) :: n
+    real,    intent(in) :: a(n), b(n)
+    logical :: ret
+    integer :: i
+    do i=1,n
+      ret = (abs(a(i) - b(i)) <= epsilon(a(i)))
+    end do
+  end function
+end program

cachelab/cbsl/tests/multi_array.c

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <math.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+
+typedef unsigned char byte_t;
+
+extern void compress(uint64_t num_vars, uint64_t data_size, byte_t** a);
+extern void raw_write(uint64_t num_vars, uint64_t data_size, byte_t** a);
+extern void decompress(uint64_t num_vars, uint64_t data_size, byte_t** a);
+extern void raw_read(uint64_t num_vars, uint64_t data_size, byte_t** a);
+extern void rand_byte_t(uint64_t data_size, byte_t* a);
+
+char cname[128], sname[128];
+
+int main(int argc, char** argv)
+{
+  if (argc <= 2)
+    exit(1);
+
+  uint64_t data_size, num_vars;
+  sscanf(argv[1], "%lu", &num_vars);
+  sscanf(argv[2], "%lu", &data_size);
+  data_size = data_size / num_vars;
+
+  printf("variables = %lu\n", num_vars);
+  printf("data size = %lf [MiB] * %lu\n", (double)(data_size)/pow(2,20), num_vars);
+
+  sprintf(cname, "multiple_compressed_%lu.zst", data_size * num_vars);
+  sprintf(sname, "multiple_raw_%lu.dat", data_size);
+
+  srand((unsigned int)(time(NULL)));
+
+  byte_t** a = malloc(sizeof(byte_t*) * num_vars);
+  byte_t** b = malloc(sizeof(byte_t*) * num_vars);
+  byte_t** c = malloc(sizeof(byte_t*) * num_vars);
+  for(uint64_t i = 0; i < num_vars; ++i)
+  {
+    a[i] = (byte_t*)(malloc(data_size));
+    rand_byte_t(data_size / sizeof(byte_t), a[i]);
+    b[i] = (byte_t*)(malloc(data_size));
+    c[i] = (byte_t*)(malloc(data_size));
+  }
+
+  compress(num_vars, data_size, a);
+  raw_write(num_vars, data_size, a);
+
+  decompress(num_vars, data_size, b);
+  raw_read(num_vars, data_size, c);
+
+  for(uint64_t i = 0; i < num_vars; ++i)
+  {
+    for(uint64_t j = 0; j < data_size; ++j)
+    {
+      if (b[i][j] != c[i][j])
+      {
+        fprintf(stderr, "mismatch!\n");
+        exit(1);
+      }
+    }
+  }
+
+  for(uint64_t i = 0; i < num_vars; ++i)
+  {
+    free(a[i]);
+    free(b[i]);
+    free(c[i]);
+  }
+  free(a);
+  free(b);
+  free(c);
+
+  return 0;
+}
+
+void compress(uint64_t num_vars, uint64_t data_size, byte_t** a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+  for(uint64_t i = 0; i < num_vars; ++i)
+    CBSL_ERROR_CHECK(cbsl_write(ctx, a[i], data_size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_write(uint64_t num_vars, uint64_t data_size, byte_t** a)
+{
+  FILE* fp = fopen(sname, "wb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen\n");
+    exit(1);
+  }
+  for(int i = 0; i < num_vars; ++i)
+    fwrite(a[i], 1, data_size, fp);
+  fclose(fp);
+}
+
+void decompress(uint64_t num_vars, uint64_t data_size, byte_t** a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+  for(uint64_t i = 0; i < num_vars; ++i)
+    CBSL_ERROR_CHECK(cbsl_read(ctx, a[i], data_size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_read(uint64_t num_vars, uint64_t data_size, byte_t** a)
+{
+  FILE* fp = fopen(sname, "rb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen\n");
+    exit(1);
+  }
+  for(uint64_t i = 0; i < num_vars; ++i)
+    fread(a[i], 1, data_size, fp);
+  fclose(fp);
+}
+
+void rand_byte_t(uint64_t data_size, byte_t* a)
+{
+  for(uint64_t i = 0; i < data_size; ++i)
+    a[i] = rand() % 255;
+}

cachelab/cbsl/tests/scalar_data.c

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+#define CHECK_BINARY(X,Y)    {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
+
+static const int    data0 = 43;
+static const double data1 = 3.14159265;
+static const int    data2 = 14142;
+
+extern void compress();
+extern void decompress();
+
+char cname[128], sname[128];
+
+int main(int argc, char** argv)
+{
+  if (argc < 2)
+    return 1;
+
+  sprintf(cname, "scalar_compressed.zst");
+  sprintf(sname, "scalar_raw.dat");
+
+  if (strcmp(argv[1],"-c") == 0)
+    compress();
+  else if (strcmp(argv[1],"-d") == 0)
+    decompress();
+  else
+    return 1;
+
+  return 0;
+}
+
+void compress()
+{
+  const int    c0 = data0;
+  const double c1 = data1;
+  const int    c2 = data2;
+
+  cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_write(ctx, &c0, sizeof(int)));
+  CBSL_ERROR_CHECK(cbsl_write(ctx, &c1, sizeof(double)));
+  CBSL_ERROR_CHECK(cbsl_write(ctx, &c2, sizeof(int)));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+
+  FILE* fp = fopen(sname, "wb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen\n");
+    exit(1);
+  }
+  fwrite(&c0, sizeof(int),    1, fp);
+  fwrite(&c1, sizeof(double), 1, fp);
+  fwrite(&c2, sizeof(int),    1, fp);
+  fclose(fp);
+}
+
+void decompress()
+{
+  int    d0, r0;
+  double d1, r1;
+  int    d2, r2;
+
+  cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_read(ctx, &d0, sizeof(int)));
+  CBSL_ERROR_CHECK(cbsl_read(ctx, &d1, sizeof(double)));
+  CBSL_ERROR_CHECK(cbsl_read(ctx, &d2, sizeof(int)));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+
+  FILE* fp = fopen(sname, "rb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen\n");
+    exit(1);
+  }
+  fread(&r0, sizeof(int),    1, fp);
+  fread(&r1, sizeof(double), 1, fp);
+  fread(&r2, sizeof(int),    1, fp);
+  fclose(fp);
+
+  CHECK_BINARY(d0, r0); CHECK_BINARY(d0, data0);
+  CHECK_BINARY(d1, r1); CHECK_BINARY(d1, data1);
+  CHECK_BINARY(d2, r2); CHECK_BINARY(d2, data2);
+}

cachelab/cbsl/tests/single_array.c

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <math.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+
+typedef unsigned char byte_t;
+
+extern void compress(uint64_t data_size, const byte_t* a);
+extern void raw_write(uint64_t data_size, const byte_t* a);
+extern void decompress(uint64_t data_size, byte_t* a);
+extern void raw_read(uint64_t data_size, byte_t* a);
+
+extern void rand_byte_t(uint64_t data_size, byte_t* a);
+
+char cname[128], sname[128];
+
+int main(int argc, char** argv)
+{
+  if (argc < 2)
+    exit(1);
+
+  int data_size;
+  sscanf(argv[1], "%d", &data_size);
+  printf("data size = %lf [MiB]\n", (double)(data_size)/pow(2,20));
+
+  sprintf(cname, "simple_compressed_%d.zst", data_size);
+  sprintf(sname, "simple_raw_%d.dat", data_size);
+
+  srand((unsigned int)(time(NULL)));
+
+  byte_t* a = (byte_t*)(malloc(data_size));
+  rand_byte_t(data_size / sizeof(byte_t), a);
+  
+  compress(data_size, a);
+  raw_write(data_size, a);
+
+  byte_t* b = (byte_t*)(malloc(data_size));
+  byte_t* c = (byte_t*)(malloc(data_size));
+
+  decompress(data_size, b);
+  raw_read(data_size, c);
+
+  for (uint64_t i = 0; i < data_size; ++i)
+  {
+    if (a[i] != b[i])
+    {
+      fprintf(stderr, "mismatch!\n");
+      exit(1);
+    }
+  }
+
+  free(a);
+  free(b);
+  free(c);
+
+  return 0;
+}
+
+void compress(uint64_t data_size, const byte_t* a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open(cbsl_store_mode)\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_write(ctx, a, data_size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_write(uint64_t data_size, const byte_t* a)
+{
+  FILE* fp = fopen(sname, "wb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen(wb)\n");
+    exit(1);
+  }
+  fwrite(a, 1, data_size, fp);
+  fclose(fp);
+}
+
+void decompress(uint64_t data_size, byte_t* a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open(cbsl_load_mode)\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_read(ctx, a, data_size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_read(uint64_t data_size, byte_t* a)
+{
+  FILE* fp = fopen(sname, "rb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen(rb)\n");
+    exit(1);
+  }
+  fread(a, 1, data_size, fp);
+  fclose(fp);
+}
+
+void rand_byte_t(uint64_t data_size, byte_t* a)
+{
+  for(uint64_t i = 0; i < data_size; ++i)
+    a[i] = rand() % 255;
+}

cachelab/cbsl/tests/split_compression.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+#define CHECK_BINARY(X,Y)    {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
+
+typedef unsigned char byte_t;
+
+extern void record(cbsl_mode, int, byte_t*, int);
+extern void rand_byte_t(int n, byte_t* a);
+
+char cname[128];
+
+int main(int argc, char** argv)
+{
+  if (argc < 3)
+    return 1;
+
+  sprintf(cname, "split.zst");
+
+  int block_size, total_size;
+  sscanf(argv[1], "%d\n", &block_size);
+  sscanf(argv[2], "%d\n", &total_size);
+
+  if (total_size < block_size)
+    return 2;
+
+  byte_t* a = malloc(sizeof(byte_t) * total_size);
+  byte_t* b = malloc(sizeof(byte_t) * total_size);
+  rand_byte_t(total_size, a);
+
+  record(cbsl_store_mode, total_size, a, block_size);
+  record(cbsl_load_mode,    total_size, b, block_size);
+
+  for (int i = 0; i < total_size; ++i)
+  {
+    if (a[i] != b[i])
+    {
+      fprintf(stderr, "mismatch!\n");
+      exit(1);
+    }
+  }
+
+  return 0;
+}
+
+void record(cbsl_mode mode, int total_size, byte_t* a, int block_size)
+{
+  cbsl_ctx* ctx = cbsl_open(mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+
+  int total = 0;
+  do
+  {
+    int size = (total_size - total < block_size) ? total_size - total : block_size;
+    CBSL_ERROR_CHECK(cbsl_record(ctx, a + total, size));
+    total += size;
+  }
+  while(total < total_size);
+  if (total != total_size)
+  {
+    fprintf(stderr, "total != total_size\n");
+    exit(1);
+  }
+
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void rand_byte_t(int n, byte_t* a) {
+  for(int i = 0; i < n; ++i)
+    a[i] = rand() % 255;
+}

cachelab/cbsl/tests/utility.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+#define CHECK(X)             {if (!(X)) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+
+char cname[128];
+
+int main(int argc, char** argv)
+{
+  sprintf(cname, "data.zst");
+  cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open\n");
+    exit(1);
+  }
+
+  CHECK(cbsl_get_mode(ctx) == cbsl_store_mode);
+  CBSL_ERROR_CHECK(cbsl_set_compression_level(ctx, 20));
+  CHECK(cbsl_get_compression_level(ctx) == 20);
+
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+
+  return 0;
+}

cachelab/cbsl/tests/variable_size_array.c

@@ -0,0 +1,155 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <math.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+
+typedef unsigned char byte_t;
+
+extern void compress(uint64_t data_size, const byte_t* a);
+extern void raw_write(uint64_t data_size, const byte_t* a);
+extern void decompress(uint64_t* data_size, byte_t** a);
+extern void raw_read(uint64_t* data_size, byte_t** a);
+
+extern uint64_t rand_size();
+extern void     rand_byte_t(uint64_t data_size, byte_t* a);
+
+char cname[128], sname[128];
+
+int main(int argc, char** argv)
+{
+  sprintf(cname, "varsize_compressed.zst");
+  sprintf(sname, "varsize_raw.dat");
+
+  srand((unsigned int)(time(NULL)));
+
+  uint64_t data_size;
+  if (argc >= 2)
+  {
+    sscanf(argv[1], "%lu\n", &data_size);
+    printf("specified data size: %lu byte\n", data_size);
+  }
+  else
+  {
+    data_size = rand_size();
+    printf("random generate data size: %lu byte\n", data_size);
+  }
+
+  byte_t* a = (byte_t*)(malloc(data_size));
+  rand_byte_t(data_size / sizeof(byte_t), a);
+  
+  compress(data_size, a);
+  raw_write(data_size, a);
+
+  uint64_t b_data_size; byte_t* b;
+  uint64_t c_data_size; byte_t* c;
+
+  decompress(&b_data_size, &b);
+  raw_read(&c_data_size, &c);
+
+  if (data_size != b_data_size || b_data_size != c_data_size)
+  {
+    fprintf(stderr, "data size is mismatch!\n");
+    exit(1);
+  }
+
+  for (uint64_t i = 0; i < data_size; ++i)
+  {
+    if (a[i] != b[i] || b[i] != c[i])
+    {
+      fprintf(stderr, "data value is mismatch!\n");
+      exit(1);
+    }
+  }
+
+  free(a);
+  free(b);
+  free(c);
+
+  return 0;
+}
+
+void compress(uint64_t data_size, const byte_t* a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open(cbsl_store_mode)\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_write(ctx, &data_size, sizeof(data_size)));
+  CBSL_ERROR_CHECK(cbsl_write(ctx, a, data_size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_write(uint64_t data_size, const byte_t* a)
+{
+  FILE* fp = fopen(sname, "wb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen(wb)\n");
+    exit(1);
+  }
+  fwrite(&data_size, 1, sizeof(data_size), fp);
+  fwrite(a, 1, data_size, fp);
+  fclose(fp);
+}
+
+void decompress(uint64_t* data_size, byte_t** a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open(cbsl_load_mode)\n");
+    exit(1);
+  }
+  CBSL_ERROR_CHECK(cbsl_read(ctx, data_size, sizeof(data_size)));
+  *a = (byte_t*)(malloc(sizeof(byte_t) * *data_size));
+  CBSL_ERROR_CHECK(cbsl_read(ctx, *a, *data_size));
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_read(uint64_t* data_size, byte_t** a)
+{
+  FILE* fp = fopen(sname, "rb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen(rb)\n");
+    exit(1);
+  }
+  fread(data_size, 1, sizeof(data_size), fp);
+  *a = (byte_t*)(malloc(sizeof(byte_t) * *data_size));
+  fread(*a, 1, *data_size, fp);
+  fclose(fp);
+}
+
+uint64_t rand_size()
+{
+  const uint64_t min_data_size = 1024;
+  const uint64_t max_data_size = 1048576; /* 1 MiB */
+  uint64_t size = rand() % max_data_size;
+  return (size < min_data_size) ? min_data_size : size;
+}
+
+void rand_byte_t(uint64_t data_size, byte_t* a)
+{
+  for(uint64_t i = 0; i < data_size; ++i)
+    a[i] = rand() % 255;
+}

cachelab/cbsl/tests/various_size_array.c

@@ -0,0 +1,189 @@
+/*
+ * Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <math.h>
+#include <cbsl.h>
+
+#define CBSL_ERROR_CHECK(X)  {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
+
+typedef unsigned char byte_t;
+
+extern void compress(uint64_t n, uint64_t* sizes, byte_t** a);
+extern void raw_write(uint64_t n, uint64_t* sizes, byte_t** a);
+extern void decompress(uint64_t n, uint64_t* sizes, byte_t** a);
+extern void raw_read(uint64_t n, uint64_t* sizes, byte_t** a);
+
+extern uint64_t rand_size();
+extern void     rand_byte_t(uint64_t data_size, byte_t* a);
+
+char cname[128], sname[128];
+
+int main(int argc, char** argv)
+{
+  sprintf(cname, "various_compressed.zst");
+  sprintf(sname, "various_raw.dat");
+
+  srand((unsigned int)(time(NULL)));
+
+  uint64_t num_vars;
+  if (argc < 2)
+  {
+    num_vars = rand() % 100 + 2;
+  }
+  else
+  {
+    sscanf(argv[1], "%lu\n", &num_vars);
+  }
+  printf("number of array (data sets): %lu\n", num_vars);
+
+  byte_t**  a = (byte_t**)(malloc(sizeof(byte_t**) * num_vars));
+  byte_t**  b = (byte_t**)(malloc(sizeof(byte_t**) * num_vars));
+  byte_t**  c = (byte_t**)(malloc(sizeof(byte_t**) * num_vars));
+  uint64_t* asizes = (uint64_t*)(malloc(sizeof(uint64_t*) * num_vars));
+  uint64_t* bsizes = (uint64_t*)(malloc(sizeof(uint64_t*) * num_vars));
+  uint64_t* csizes = (uint64_t*)(malloc(sizeof(uint64_t*) * num_vars));
+
+  for(uint64_t i = 0; i < num_vars; ++i)
+  {
+    asizes[i] = rand_size();
+    a[i] = (byte_t*)(malloc(sizeof(byte_t*) * asizes[i]));
+    rand_byte_t(asizes[i] / sizeof(byte_t), a[i]);
+    printf("array[%lu] = %lu bytes\n", i, asizes[i]);
+
+    b[i] = (byte_t*)(malloc(sizeof(byte_t*) * asizes[i]));
+    c[i] = (byte_t*)(malloc(sizeof(byte_t*) * asizes[i]));
+  }
+  
+  compress(num_vars, asizes, a);
+  raw_write(num_vars, asizes, a);
+
+  decompress(num_vars, bsizes, b);
+  raw_read(num_vars, csizes, c);
+
+  for (uint64_t i = 0; i < num_vars; ++i)
+  {
+    if (asizes[i] != bsizes[i] || bsizes[i] != csizes[i])
+    {
+      fprintf(stderr, "array size is mismatch!\n");
+      exit(1);
+    }
+  }
+
+  for (uint64_t i = 0; i < num_vars; ++i)
+  for (uint64_t j = 0; j < asizes[i]; ++j)
+  {
+    if (a[i][j] != b[i][j] || b[i][j] != c[i][j])
+    {
+      fprintf(stderr, "data value is mismatch! (a,b,c)[%lu][%lu]\n", i, j);
+      exit(1);
+    }
+  }
+
+  for (uint64_t i = 0; i < num_vars; ++i)
+  {
+    free(a[i]);
+    free(b[i]);
+    free(c[i]);
+  }
+  free(a);
+  free(b);
+  free(c);
+  free(asizes);
+  free(bsizes);
+  free(csizes);
+
+  return 0;
+}
+
+void compress(uint64_t n, uint64_t* sizes, byte_t** a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open(cbsl_store_mode)\n");
+    exit(1);
+  }
+  for (uint64_t i = 0; i < n; ++i)
+  {
+    CBSL_ERROR_CHECK(cbsl_write(ctx, &sizes[i], sizeof(sizes[i])));
+    CBSL_ERROR_CHECK(cbsl_write(ctx, a[i], sizes[i]));
+  }
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_write(uint64_t n, uint64_t* sizes, byte_t** a)
+{
+  FILE* fp = fopen(sname, "wb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen(wb)\n");
+    exit(1);
+  }
+  for (uint64_t i = 0; i < n; ++i)
+  {
+    fwrite(&sizes[i], 1, sizeof(sizes[i]), fp);
+    fwrite(a[i], 1, sizes[i], fp);
+  }
+  fclose(fp);
+}
+
+void decompress(uint64_t n, uint64_t* sizes, byte_t** a)
+{
+  cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
+  if (ctx == NULL)
+  {
+    fprintf(stderr, "error: cbsl_open(cbsl_load_mode)\n");
+    exit(1);
+  }
+  for (uint64_t i = 0; i < n; ++i)
+  {
+    CBSL_ERROR_CHECK(cbsl_read(ctx, &sizes[i], sizeof(sizes[i])));
+    CBSL_ERROR_CHECK(cbsl_read(ctx, a[i], sizes[i]));
+  }
+  CBSL_ERROR_CHECK(cbsl_close(ctx));
+}
+
+void raw_read(uint64_t n, uint64_t* sizes, byte_t** a)
+{
+  FILE* fp = fopen(sname, "rb");
+  if (fp == NULL)
+  {
+    fprintf(stderr, "error: fopen(rb)\n");
+    exit(1);
+  }
+  for (uint64_t i = 0; i < n; ++i)
+  {
+    fread(&sizes[i], 1, sizeof(sizes[i]), fp);
+    fread(a[i], 1, sizes[i], fp);
+  }
+  fclose(fp);
+}
+
+uint64_t rand_size()
+{
+  const uint64_t min_data_size = 1024;
+  const uint64_t max_data_size = 1048576; /* 1 MiB */
+  uint64_t size = rand() % max_data_size;
+  return (size < min_data_size) ? min_data_size : size;
+}
+
+void rand_byte_t(uint64_t data_size, byte_t* a)
+{
+  for(uint64_t i = 0; i < data_size; ++i)
+    a[i] = rand() % 255;
+}