gh0s7/csapp2025
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cachelab started

Browse Source
This commit is contained in:
CGH0S7
2025-04-21 23:52:27 +08:00
parent cc99d9b5d9
commit ace7a46fb9
58 changed files with 10071 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
cachelab/cbsl/CMakeLists.txt Normal file
View File

@@ -0,0 +1,125 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
cmake_minimum_required(VERSION 3.3)
enable_testing()
set(CMAKE_C_STANDARD 99)
project(cbsl
VERSION 2019.5.0
LANGUAGES C Fortran
)
if (CMAKE_BUILD_TYPE)
string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
if (NOT uppercase_CMAKE_BUILD_TYPE MATCHES "^(DEBUG|RELEASE|RELWITHDEBINFO|MINSIZEREL)$")
message(FATAL_ERROR "Invalid value for CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
endif()
if (${uppercase_CMAKE_BUILD_TYPE} MATCHES "DEBUG")
set(CBSL_DEBUG 1)
endif ()
endif ()
if (INSTALL_ZSTD)
message(STATUS "Enable installation Zstandard library version 1.4.0")
include(ExternalProject)
ExternalProject_Add(zstd
GIT_REPOSITORY "https://github.com/facebook/zstd.git"
GIT_TAG "v1.4.0"
PREFIX "${CMAKE_BINARY_DIR}/zstd"
SOURCE_SUBDIR "build/cmake"
CMAKE_ARGS -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -D CMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
STEP_TARGETS install
EXCLUDE_FROM_ALL on
)
include_directories(${CMAKE_INSTALL_PREFIX}/include)
link_directories(${CMAKE_INSTALL_PREFIX}/lib)
endif ()
find_library(ZSTD_LIB NAMES "zstd")
include(CheckCCompilerFlag)
CHECK_C_COMPILER_FLAG("-Wall" C_HAS_WALL)
CHECK_C_COMPILER_FLAG("-Wshadow" C_HAS_WSHADOW)
CHECK_C_COMPILER_FLAG("-Werror" C_HAS_WERROR)
CHECK_C_COMPILER_FLAG("-pedantic-errors" C_HAS_PEDANTIC_ERRORS)
set(ADD_C_EXTRA_FLAGS)
if (C_HAS_WALL)
set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -Wall")
endif ()
if (C_HAS_WSHADOW)
set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -Wshadow")
endif ()
if (C_HAS_WERROR)
set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -Werror")
endif ()
if (C_HAS_PEDANTIC_ERRORS)
set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -pedantic-errors")
endif ()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ADD_C_EXTRA_FLAGS}")
include(CheckFortranCompilerFlag)
CHECK_Fortran_COMPILER_FLAG("-Wall -Wextra" Fortran_HAS_WALL_EXTRA)
CHECK_Fortran_COMPILER_FLAG("-Wshadow" Fortran_HAS_WERROR)
CHECK_Fortran_COMPILER_FLAG("-pedantic-errors" Fortran_HAS_PEDANTIC_ERRORS)
set(ADD_Fortran_EXTRA_FLAGS)
if (Fortran_HAS_WALL_EXTRA)
set(ADD_Fortran_EXTRA_FLAGS "${ADD_Fortran_EXTRA_FLAGS} -Wall -Wextra")
endif ()
if (Fortran_HAS_WERROR)
set(ADD_Fortran_EXTRA_FLAGS "${ADD_Fortran_EXTRA_FLAGS} -Werror")
endif ()
if (Fortran_HAS_PEDANTIC_ERRORS)
set(ADD_Fortran_EXTRA_FLAGS "${ADD_Fortran_EXTRA_FLAGS} -pedantic-errors")
endif ()
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${ADD_Fortran_EXTRA_FLAGS}")
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(${PROJECT_BINARY_DIR}/include)
set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/include)
set(CBSL_LIB "cbsl")
set(CBSL_FLIB "${CBSL_LIB}f")
add_subdirectory(include)
add_subdirectory(src)
add_subdirectory(examples)
add_subdirectory(tests)
add_subdirectory(benchmarks)
if (INSTALL_ZSTD)
# install zstd before building the library
add_dependencies(${CBSL_FLIB} ${CBSL_LIB})
add_dependencies(${CBSL_LIB} zstd-install)
endif ()

202
cachelab/cbsl/LICENSE Normal file
View File

@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

93
cachelab/cbsl/README.md Normal file
View File

@@ -0,0 +1,93 @@
# CBSL: Compressed Binary Serialization Library
## What's this?
The library provides the binary serialization with the compression by [Zstandard](https://facebook.github.io/zstd/).
A motivation of the library is to implement the data-compressed checkpoint/restart, which is well-known technique to recover computer failures in high-performance computing.
This library aims to simple and lightweight access for users.
The library support C99 or later, and Fortran 2008 or later: perhaps we use the features are all supported by major compilers.
## How to build and test
The library uses [CMake](https://cmake.org/) version 3.3.x or later.
$ cmake --version
cmake version 3.14.3
CMake suite maintained and supported by Kitware (kitware.com/cmake).
$ mkdir build && cd build
$ cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/install/path ..
$ make
$ make test
$ make install
## Do you need help to install `zstd` package?
We can build and install `zstd` automatically in the build process.
Please pass `-D INSTALL_ZSTD=on` to cmake, we will install `zstd` package where `CMAKE_INSTALL_PREFIX` directory before building the library.
$ cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/install/path -D INSTALL_ZSTD=on ..
$ make
$ make test
$ make install
## Test environments
1. CMake version 3.14.3
2. GCC version 4.8.5
3. Zstandard version 1.4.0
4. CentOS Linux release 7.5.1804 (Core)
## Benchmark
We provide the library performance benchmark with best and worst case.
Please be reminded that **the benchmark results not indicate the performance of Zstandard**, these measure the overhead (use cost) of this library.
1. Best case : all data is zero filled (A compression ratio achieves up to 99%)
2. Worst case : data is generated by rand() (A compression ratio is lower than 1%)
`benchmark` target executes the benchmarks.
...
$ make benchmark
min data size = 262144.00 [B]
max data size = 134217728.00 [B]
data is zero filled (maximum compression)
<write data [Byte]> <time [seconds]> <speed [MiB/sec]>
262144 0.002397 109.360615
524288 0.002887 181.572393
1048576 0.003440 304.827177
...
## Use `zstd` command
A compressed file by the library can be decompressed by `zstd` command.
## License
Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
## Future work
1. API error code
2. Refectoring tests
## NOTE
### Unsupported binary compatibility
The library does not support the binary compatibility and endianness conversion, which is required on the communication across machine.
If you want it, please consider using other serialization libraries such as [MessagePack](https://msgpack.org/).

46
cachelab/cbsl/benchmarks/CMakeLists.txt Normal file
View File

@@ -0,0 +1,46 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include(CheckLibraryExists)
CHECK_LIBRARY_EXISTS(rt clock_gettime "time.h" HAVE_CLOCK_GETTIME)
if (HAVE_CLOCK_GETTIME)
link_libraries(rt)
else ()
include(CheckFunctionExists)
CHECK_FUNCTION_EXISTS(gettimeofday HAVE_GETTIMEOFDAY)
if (not HAVE_GETTIMEOFDAY)
message(WARNING "Benchmark program uses low resolution timer...")
endif ()
endif ()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)
link_libraries(${CBSL_LIB} ${ZSTD_LIB} m)
include_directories(${CMAKE_CURRENT_BINARY_DIR})
add_executable(serialize serialize.c)
add_executable(deserialize deserialize.c)
add_custom_target(benchmark_serialize_bestcase ./serialize 18 27 -f DEPENDS serialize)
add_custom_target(benchmark_deserialize_bestcase ./deserialize 18 27 -f DEPENDS deserialize benchmark_serialize_bestcase)
add_custom_target(benchmark_fast DEPENDS benchmark_serialize_bestcase benchmark_deserialize_bestcase)
add_custom_target(benchmark_serialize_worstcase ./serialize 18 27 -r DEPENDS serialize)
add_custom_target(benchmark_deserialize_worstcase ./deserialize 18 27 -r DEPENDS deserialize benchmark_serialize_worstcase)
add_custom_target(benchmark_slow DEPENDS benchmark_serialize_worstcase benchmark_deserialize_worstcase)
add_custom_target(benchmark DEPENDS benchmark_fast benchmark_slow)

22
cachelab/cbsl/benchmarks/config.h.in Normal file
View File

@@ -0,0 +1,22 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CBSL_BENCHMARK_CONFIG_INCLUDED
#define CBSL_BENCHMARK_CONFIG_INCLUDED
#cmakedefine01 HAVE_CLOCK_GETTIME
#cmakedefine01 HAVE_GETTIMEOFDAY
#endif /* CBSL_BENCHMARK_CONFIG_INCLUDED */

99
cachelab/cbsl/benchmarks/deserialize.c Normal file
View File

@@ -0,0 +1,99 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#ifdef HAVE_GETTIMEOFDAY
#include <sys/time.h>
#endif
#include <math.h>
#include <cbsl.h>
#include "config.h"
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
typedef unsigned char byte_t;
extern void deserialize_bench(double ds);
extern double seconds();
int main(int argc, char** argv)
{
if (argc <= 2)
exit(1);
int max_data_size_exp2, min_data_size_exp2;
sscanf(argv[1], "%d", &min_data_size_exp2);
sscanf(argv[2], "%d", &max_data_size_exp2);
printf("min data size = %.2lf [B]\n", pow(2,min_data_size_exp2));
printf("max data size = %.2lf [B]\n", pow(2,max_data_size_exp2));
srand((unsigned int)(time(NULL)));
printf("<read data [Byte]> <time [seconds]> <speed [MiB/sec]>\n");
for (int i = min_data_size_exp2; i <= max_data_size_exp2; ++i)
{
deserialize_bench(pow(2,i));
}
return 0;
}
void deserialize_bench(double ds)
{
const uint64_t data_size = (uint64_t)(ds);
char cname[128];
sprintf(cname, "serialize_%lu.zst", data_size);
byte_t* a = (byte_t*)(malloc(data_size));
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
const double beg = seconds();
{
CBSL_ERROR_CHECK(cbsl_read(ctx, a, data_size));
}
const double end = seconds();
const double rt = end - beg;
CBSL_ERROR_CHECK(cbsl_close(ctx));
printf("%lu %lf %lf\n", data_size, rt, (data_size/rt)/1.0e6);
free(a);
}
double seconds() {
#if defined(HAVE_CLOCK_GETTIME)
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
return ts.tv_sec + (ts.tv_nsec / 1.0e9);
#elif defined(HAVE_GETTIMEOFDAY)
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec + (ts.tv_usec / 1.0e6);
#else
return (double)(clock() / CLOCKS_PER_SEC);
#endif
}

136
cachelab/cbsl/benchmarks/serialize.c Normal file
View File

@@ -0,0 +1,136 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <string.h>
#ifdef HAVE_GETTIMEOFDAY
#include <sys/time.h>
#endif
#include <math.h>
#include <cbsl.h>
#include "config.h"
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
typedef unsigned char byte_t;
extern void serialize_bench(double ds);
extern void rand_byte_t(uint64_t n, byte_t* a);
extern void zero_filled(uint64_t n, byte_t* a);
extern double seconds();
int rand_generate = 0;
int main(int argc, char** argv)
{
if (argc < 4)
exit(1);
int max_data_size_exp2, min_data_size_exp2;
sscanf(argv[1], "%d", &min_data_size_exp2);
sscanf(argv[2], "%d", &max_data_size_exp2);
printf("min data size = %.2lf [B]\n", pow(2,min_data_size_exp2));
printf("max data size = %.2lf [B]\n", pow(2,max_data_size_exp2));
rand_generate = (strcmp(argv[3],"-r") == 0);
if (rand_generate)
{
printf("data is random generated (stressful compression)\n");
}
else
{
printf("data is zero filled (maximum compression)\n");
}
srand((unsigned int)(time(NULL)));
printf("<write data [Byte]> <time [seconds]> <speed [MiB/sec]>\n");
for (int i = min_data_size_exp2; i <= max_data_size_exp2; ++i)
{
serialize_bench(pow(2,i));
}
return 0;
}
void serialize_bench(double ds)
{
const uint64_t data_size = (uint64_t)(ds);
char cname[128];
sprintf(cname, "serialize_%lu.zst", data_size);
byte_t* a = (byte_t*)(malloc(data_size));
if (rand_generate)
{
rand_byte_t(data_size / sizeof(byte_t), a);
}
else
{
zero_filled(data_size / sizeof(byte_t), a);
}
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
const double beg = seconds();
{
CBSL_ERROR_CHECK(cbsl_write(ctx, a, data_size));
CBSL_ERROR_CHECK(cbsl_flush(ctx));
}
const double end = seconds();
const double wt = end - beg;
CBSL_ERROR_CHECK(cbsl_close(ctx));
printf("%lu %lf %lf\n", data_size, wt, (data_size/wt)/1.0e6);
free(a);
}
void rand_byte_t(uint64_t data_size, byte_t* a)
{
for(uint64_t i = 0; i < data_size; ++i)
a[i] = rand() % 255;
}
void zero_filled(uint64_t n, byte_t* a)
{
for(uint64_t i = 0; i < n; ++i)
a[i] = 0; /* all data fills zero */
}
double seconds() {
#if defined(HAVE_CLOCK_GETTIME)
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
return ts.tv_sec + (ts.tv_nsec / 1.0e9);
#elif defined(HAVE_GETTIMEOFDAY)
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec + (ts.tv_usec / 1.0e6);
#else
return (double)(clock() / CLOCKS_PER_SEC);
#endif
}

22
cachelab/cbsl/examples/CMakeLists.txt Normal file
View File

@@ -0,0 +1,22 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
add_executable(simple_usage simple_usage.c)
add_dependencies(simple_usage ${CBSL_LIB})
target_link_libraries(simple_usage ${CBSL_LIB} ${ZSTD_LIB})
add_executable(recommend_usage recommend_usage.c)
add_dependencies(recommend_usage ${CBSL_LIB})
target_link_libraries(recommend_usage ${CBSL_LIB} ${ZSTD_LIB})

101
cachelab/cbsl/examples/recommend_usage.c Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
#define CHECK_BINARY(X,Y) {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
#define CHECK(X) {if (!(X)) { fprintf(stderr, "error: %s\n", (#X)); }}
#define SAFE_FREE(X) {if ((X) != NULL) { free((X)); }}
static const int data0 = 43;
static const double data1 = 3.14159265;
static const int data2 = 14142;
extern void checkpoint_restart(cbsl_mode, int*, double*, int*, int**);
char cname[128];
int main(int argc, char** argv)
{
int c0;
double c1;
int c2;
int* a0;
if (argc < 2)
return 1;
sprintf(cname, "checkpoint.zst");
cbsl_mode mode = cbsl_unknown_mode;
if (strcmp(argv[1],"-c") == 0)
{
mode = cbsl_store_mode;
c0 = data0;
c1 = data1;
c2 = data2;
a0 = (int*)(malloc(sizeof(int) * 42));
for (int i = 0; i < 42; ++i)
a0[i] = 42;
}
else if (strcmp(argv[1],"-r") == 0)
{
mode = cbsl_load_mode;
c0 = c1 = c2 = 0;
a0 = NULL;
}
else
{
return 1;
}
checkpoint_restart(mode, &c0, &c1, &c2, &a0);
CHECK_BINARY(c0, data0);
CHECK_BINARY(c1, data1);
CHECK_BINARY(c2, data2);
for (int i = 0; i < 42; ++i)
{
CHECK(a0[i] == 42);
}
SAFE_FREE(a0);
return 0;
}
void checkpoint_restart(cbsl_mode mode, int* c0, double* c1, int* c2, int** a0)
{
int a0_size;
cbsl_ctx* ctx = cbsl_open(mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_record(ctx, c0, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_record(ctx, c1, sizeof(double)));
CBSL_ERROR_CHECK(cbsl_record(ctx, c2, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_record(ctx, &a0_size, sizeof(int)));
if (mode == cbsl_load_mode)
(*a0) = (int*)(malloc(sizeof(int) * a0_size));
CBSL_ERROR_CHECK(cbsl_record(ctx, (*a0), sizeof(int) * a0_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}

99
cachelab/cbsl/examples/simple_usage.c Normal file
View File

@@ -0,0 +1,99 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <string.h>
#include <cbsl.h>
#define ARRAY_SIZE(X) (sizeof((X))/sizeof(*(X)))
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); }}
int a[1024]; /* 4 KiB */
double b[1024]; /* 8 KiB */
extern void save(void);
extern void load(void);
extern void rand_int(int, int*);
extern void rand_double(int, double*);
char cname[128];
int main(int argc, char** argv) {
srand((unsigned int)(time(NULL)));
sprintf(cname, "checkpoint.zst");
if (strcmp(argv[1],"-c") == 0)
{
rand_int(ARRAY_SIZE(a), a);
rand_double(ARRAY_SIZE(b), b);
save();
}
else if (strcmp(argv[1],"-r") == 0)
{
load();
}
else
{
return 1;
}
return 0;
}
void save(void) {
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, "./checkpoint.data");
if (ctx == NULL) {
fprintf(stderr, "error: cbsl_open save\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_write(ctx, a, sizeof(a)));
CBSL_ERROR_CHECK(cbsl_write(ctx, b, sizeof(b)));
CBSL_ERROR_CHECK(cbsl_close(ctx));
printf("before compressed a[%d] = %d\n", 124, a[124]);
printf("before compressed b[%d] = %e\n", 514, b[514]);
}
void load(void) {
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, "./checkpoint.data");
if (ctx == NULL) {
fprintf(stderr, "error: cbsl_open load\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_read(ctx, a, sizeof(a)));
CBSL_ERROR_CHECK(cbsl_read(ctx, b, sizeof(b)));
CBSL_ERROR_CHECK(cbsl_close(ctx));
printf("decompressed a[%d] = %d\n", 124, a[124]);
printf("decompressed b[%d] = %e\n", 514, b[514]);
}
void rand_int(int n, int *v) {
for(int i = 0; i < n; ++i)
v[i] = rand();
}
void rand_double(int n, double *v) {
for(int i = 0; i < n; ++i)
v[i] = 1.0 / (double)(rand());
}

23
cachelab/cbsl/include/CMakeLists.txt Normal file
View File

@@ -0,0 +1,23 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cbsl_config.h.in
${CMAKE_CURRENT_BINARY_DIR}/cbsl_config.h)
install(FILES
${CMAKE_CURRENT_SOURCE_DIR}/cbsl.h
${CMAKE_CURRENT_BINARY_DIR}/cbsl_config.h
TYPE INCLUDE
)

63
cachelab/cbsl/include/cbsl.h Normal file
View File

@@ -0,0 +1,63 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CBSL_HEADER_INCLUDED
#define CBSL_HEADER_INCLUDED
#if defined(__cplusplus)
extern "C" {
#endif
#include <stdint.h>
#include "cbsl_config.h"
typedef enum
{
cbsl_load_mode = 1,
cbsl_store_mode = 2,
cbsl_unknown_mode = -1
}
cbsl_mode;
typedef enum
{
cbsl_success = 0,
cbsl_end = 1,
cbsl_error = -1
}
cbsl_errors;
typedef struct cbsl_ctx_ cbsl_ctx;
cbsl_ctx* cbsl_open(cbsl_mode open_mode, char* path);
cbsl_errors cbsl_close(cbsl_ctx* ctx);
cbsl_errors cbsl_flush(cbsl_ctx* ctx);
cbsl_errors cbsl_write(cbsl_ctx* ctx, const void* data, uint64_t size);
cbsl_errors cbsl_read(cbsl_ctx* ctx, void* data, uint64_t size);
cbsl_errors cbsl_readline(cbsl_ctx* ctx, char* linebuf, uint64_t size);
cbsl_errors cbsl_record(cbsl_ctx* ctx, void* data, uint64_t size);
cbsl_errors cbsl_record_heap(cbsl_ctx* ctx, void** data, uint64_t* size);
cbsl_mode cbsl_get_mode(cbsl_ctx* ctx);
cbsl_errors cbsl_set_compression_level(cbsl_ctx* ctx, int clevel);
int cbsl_get_compression_level(cbsl_ctx* ctx);
#if defined(__cplusplus)
} /* extern "C" */
#endif
#endif /* CBSL_HEADER_INCLUDED */

27
cachelab/cbsl/include/cbsl_config.h Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CBSL_CONFIG_HEADER_INCLUDED
#define CBSL_CONFIG_HEADER_INCLUDED
#define CBSL_VERSION 201950
#define CBSL_VERSION_STRING "2019.5.0"
#define CBSL_MAJOR_VERSION 2019
#define CBSL_MINOR_VERSION 5
#define CBSL_PATCH_VERSION 0
/* #undef CBSL_DEBUG */
#endif /* CBSL_CONFIG_HEADER_INCLUDED */

27
cachelab/cbsl/include/cbsl_config.h.in Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CBSL_CONFIG_HEADER_INCLUDED
#define CBSL_CONFIG_HEADER_INCLUDED
#define CBSL_VERSION @cbsl_VERSION_MAJOR@@cbsl_VERSION_MINOR@@cbsl_VERSION_PATCH@
#define CBSL_VERSION_STRING "@cbsl_VERSION@"
#define CBSL_MAJOR_VERSION @cbsl_VERSION_MAJOR@
#define CBSL_MINOR_VERSION @cbsl_VERSION_MINOR@
#define CBSL_PATCH_VERSION @cbsl_VERSION_PATCH@
#cmakedefine CBSL_DEBUG
#endif /* CBSL_CONFIG_HEADER_INCLUDED */

443
cachelab/cbsl/references.md Normal file
View File

@@ -0,0 +1,443 @@
# CBSL library references
# Build and link libraries
## C
```bash
export CBSL_INCLUDE_PATH=${CBSL_INSTALLED_PATH}/include
export CBSL_LIBRARY_PATH=${CBSL_INSTALLED_PATH}/lib
cc -c xcbsl_use_code.c -I${CBSL_INCLUDE_PATH}
cc -o xcbsl_use_program ${OBJS} -lcbsl -lzstd -L${CBSL_LIBRARY_PATH}
```
## Fortran
```bash
export CBSL_INCLUDE_PATH=${CBSL_INSTALLED_PATH}/include
export CBSL_LIBRARY_PATH=${CBSL_INSTALLED_PATH}/lib
f95 -c xcbslf_use_code.f95 -I${CBSL_INCLUDE_PATH}
f95 -o xcbslf_use_program ${OBJS} -lcbslf -lcbsl -lzstd -L${CBSL_LIBRARY_PATH}
```
# API
We provide `cbslf` module for using the library in Fortran application.
## Versioning
|Version|Value |
|-------|-----------------------|
|major |release year (ex. 2019)|
|minor |release month (ex. 5) |
|patch |patch version (0-9) |
### C
```c
#define CBSL_VERSION // full version (integer)
#define CBSL_VERSION_STRING // full version (string)
#define CBSL_MAJOR_VERSION // major
#define CBSL_MINOR_VERSION // minor
#define CBSL_PATCH_VERSION // patch
```
### Fortran
```fortran
integer(8), parameter :: CBSL_VERSION
character(*), parameter :: CBSL_VERSION_STRING
integer(4), parameter :: CBSL_MAJOR_VERSION
integer(4), parameter :: CBSL_MINOR_VERSION
integer(4), parameter :: CBSL_PATCH_VERSION
```
## Types
### C
```c
// library context
typedef struct cbsl_ctx_ cbsl_ctx;
```
### Fortran
```fortran
type(cbslf_context)
```
## Constant values
### C
```c
// Context mode
typedef enum
{
cbsl_load_mode, // deserialize mode
cbsl_store_mode, // serialize mode
cbsl_unknown_mode // unknown
}
cbsl_mode;
// Errors
typedef enum
{
cbsl_success, // no error
cbsl_error // function happens any error
}
cbsl_errors;
```
### Fortran
```fortran
integer(4), parameter :: cbslf_load_mode
integer(4), parameter :: cbslf_store_mode
integer(4), parameter :: cbslf_unknown_mode
integer(4), parameter :: cbslf_success
integer(4), parameter :: cbslf_errors
```
## API
### C
```c
/*
[brief]
open serialized data file stream.
[arguments]
open_mode: specifies load/store mode
path : data file path
[return]
success : return context pointer
fail : NULL pointer
*/
cbsl_ctx* cbsl_open(cbsl_mode open_mode, char* path);
/*
[brief]
close serialized data file stream.
this function calls cbsl_flush() before closing stream.
[arguments]
ctx : context pointer
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_close(cbsl_ctx* ctx);
/*
[brief]
flush file stream.
this function executes data compression and write to file stream.
[arguments]
ctx : context pointer
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_flush(cbsl_ctx* ctx);
/*
[brief]
the data stores with compression.
this function stores internal buffer to stored data.
When the stored data fills enough size to compress, this function calls cbsl_flush().
[arguments]
ctx : context pointer
data : write data pointer (must be allocated)
size : byte size of data
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_write(cbsl_ctx* ctx, const void* data, uint64_t size);
/*
[brief]
the data loads with decompression.
this function decompresses data from a file stream, and loads from internal decompressed buffer.
[arguments]
ctx : context pointer
data : read data pointer (must be allocated)
size : byte size of data
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_read(cbsl_ctx* ctx, void* data, uint64_t size);
/*
[brief]
the data loads/stores with compression/decompression.
this function calls cbsl_read()/cbsl_write() by context mode to help the implementation of checkpoint/restart in the application.
[argumetns]
ctx : context pointer
data : read/write data pointer (must be allocated)
size : byte size of data
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_record(cbsl_ctx* ctx, void* data, uint64_t size);
/*
[brief]
this is a specialized function of cbsl_record() for heap allocated array.
the function loads/stores array data size [bytes] and array values from/to a file stream.
an array will be allocated on heap memory by malloc() routine if `data` is null pointer.
[arguments]
ctx : context pointer
data : read/write data pointer (it accepts NULL pointer)
size : byte size of data
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_record_heap(cbsl_ctx* ctx, void** data, uint64_t* size);
/*
[brief]
gets context mode.
[arguments]
ctx : context pointer
[return]
success : context mode
*/
cbsl_mode cbsl_get_mode(cbsl_ctx* ctx);
/*
[brief]
sets zstd compression level of serialized data.
[arguments]
ctx : context pointer
clevel : compression level 1-22 (zstd 1.4.0)
[return]
success : cbsl_success
fail : cbsl_error
*/
cbsl_errors cbsl_set_compression_level(cbsl_ctx* ctx, int clevel);
/*
[brief]
gets zstd compression level of serialized data.
[arguments]
ctx : context pointer
[return]
success : compression level 1-22 (zstd 1.4.0)
fail : -1
*/
int cbsl_get_compression_level(cbsl_ctx* ctx);
```
### Fortran
```fortran
!
! [brief]
! open serialized data file stream.
! [arguments]
! open_mode: specifies load/store mode
! path : data file path
! errcode : error code
! [return]
! success : return context pointer
! fail : NULL pointer
!
function cbslf_open(open_mode, path, errcode) result(ctx)
integer(4), intent(in) :: open_mode
character(*), intent(in) :: path
integer(4), intent(out), optional :: errcode
type(cbslf_context) :: ctx
end function
!
! [brief]
! close serialized data file stream.
! this subroutine calls cbslf_flush() before closing stream.
! [arguments]
! ctx : context pointer
! errcode : error code
!
subroutine cbslf_close(ctx, errcode)
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
end subroutine
!
! [brief]
! flush file stream.
! this subroutine executes data compression and write to file stream.
! [arguments]
! ctx : context pointer
! errcode : error code
!
subroutine cbslf_flush(ctx, errcode)
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
end subroutine
!
! [brief]
! this is a generic interface.
! the data stores with compression.
! this subroutine stores internal buffer to stored data.
! When the stored data fills enough size to compress, this subroutine calls cbslf_flush().
! [supported types]
! scalar : logical, character(*)
! +array : integer(4), integer(8), real(4), real(8), complex(4), complex(8) by up to 7-dimensional array
! [arguments]
! ctx : context pointer
! data : write data (array must be allocated)
! errcode : error code
!
interface cbslf_write(ctx, data, errcode)
type(cbslf_context), intent(in) :: ctx
GENERIC_TYPE, intent(in) :: data
integer(4), intent(out), optional :: errcode
end interface
!
! [brief]
! this is a generic interface.
! the data loads with decompression.
! this subroutine decompresses data from a file stream, and loads from internal decompressed buffer.
! [supported types]
! scalar : logical, character(*)
! +array : integer(4), integer(8), real(4), real(8), complex(4), complex(8) by 7-dimensional array
! [arguments]
! ctx : context pointer
! data : write data (array must be allocated)
! errcode : error code
!
interface cbslf_read(ctx, data, errcode)
type(cbslf_context), intent(in) :: ctx
GENERIC_TYPE, intent(out) :: data
integer(4), intent(out), optional :: errcode
end interface
!
! [brief]
! this is a generic interface.
! the data loads/stores with compression/decompression.
! this subroutine calls cbsl_read()/cbsl_write() by context mode to help the implementation of checkpoint/restart in the application.
! [supported types]
! scalar : logical, character(*)
! +array : integer(4), integer(8), real(4), real(8), complex(4), complex(8) by up to 7-dimensional array
! [arguments]
! ctx : context pointer
! data : read/write data (array must be allocated)
! size : byte size of data
! errcode : error code (return value)
!
interface cbslf_record(ctx, data, errcode)
type(cbslf_context), intent(in) :: ctx
GENERIC_TYPE, intent(inout) :: data
integer(4), intent(out), optional :: errcode
end interface
!
! [brief]
! this is a generic interface for allocatable array.
! the data loads/stores with compression/decompression.
! this subroutine calls cbsl_read()/cbsl_write() by context mode to help the implementation of checkpoint/restart in the application.
! this subroutine behaves as like as cbsl_record_heap() in C API.
! an array will be allocated on the memory if it is not allocated.
! [supported types]
! array : integer(4), integer(8), real(4), real(8), complex(4), complex(8) by up to 7-dimensional array
! [arguments]
! ctx : context pointer
! data : read/write data (accepts not allocated array)
! size : byte size of data
! errcode : error code
!
interface cbslf_record_heap(ctx, data, errcode)
type(cbslf_context), intent(in) :: ctx
GENERIC_TYPE, intent(inout) :: data
integer(4), intent(out), optional :: errcode
end interface
!
! [brief]
! gets context mode.
! [arguments]
! ctx : context pointer
! errcode : error code (return value)
! [return]
! success : context mode
!
function cbslf_get_mode(ctx, errcode) result(mode)
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: mode
end function
!
! [brief]
! sets zstd compression level of serialized data.
! [arguments]
! ctx : context pointer
! clevel : compression level 1-22 (zstd 1.4.0)
! errcode : error code (return value)
!
subroutine cbslf_set_compression_level(ctx, clevel)
type(cbslf_context), intent(in) :: ctx
ingeter(4), intent(in) :: clevel
integer(4), intent(out), optional :: errcode
end subroutine
!
! [brief]
! gets zstd compression level of serialized data.
! [arguments]
! ctx : context pointer
! [return]
! success : compression level 1-22 (zstd 1.4.0)
! fail : -1
!
function cbslf_get_compression_level(ctx, errcode) result(clevel)
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
ingeter(4), :: clevel
end function
```
## Data format
```
BDATA: binary data [1 byte]
BSIZE: total size of binary data [4 bytes]
RSIZE: rank size of a fortran array [4 bytes]
NRANK: number of rank size of a fortran array
sizeof(TYPE): gets byte size of TYPE
Array-value-type: element type of an array
```
### Header
```
[BDATA * 8]: cbsl library version (64-bit integer)
```
### C API
```
Scalar data: HEAD -> [BDATA * sizeof(Scalar)] -> TAIL
Array data : HEAD -> [BSIZE][BDATA * sizeof(Array-value-type) * BSIZE] -> TAIL
```
### Fortran API
```
Scalar data: HEAD -> [BDATA * sizeof(Scalar)] -> TAIL
Array data : HEAD -> [BSIZE][ARRAY_LOWER_BOUNDS][ARRAY_UPPER_BOUNDS][BDATA * sizeof(Array-value-type) * BSIZE] -> TAIL
ARRAY_LOWER_BOUNDS: lower bounds of a fortran array [RSIZE * NRANK]
ARRAY_UPPER_BOUNDS: upper bounds of a fortran array [RSIZE * NRANK]
```

46
cachelab/cbsl/src/CMakeLists.txt Normal file
View File

@@ -0,0 +1,46 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set(LIB_SOURCES
buffer.c
file.c
utils.c
flush.c
write.c
read.c
record.c
)
set(FLIB_SOURCES
${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
)
configure_file(fortran_bindings.f90.in ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90.in.2)
find_program(CPP cpp)
if (CPP_NOTFOUND)
message(FATAL_ERROR "C preprocessor (cpp) command not found.")
endif ()
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
COMMAND ${CPP} ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90.in.2 ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_fortran_bindings.cmake ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90.in.2
)
add_custom_target(generate_preprocessed_fortran_file SOURCES ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90)
add_library(${CBSL_LIB} ${LIB_SOURCES})
add_library(${CBSL_FLIB} ${FLIB_SOURCES})
install(TARGETS ${CBSL_LIB} ${CBSL_FLIB})
install(FILES "${CMAKE_Fortran_MODULE_DIRECTORY}/${CBSL_FLIB}.mod" DESTINATION include)

85
cachelab/cbsl/src/buffer.c Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "./cbsl_internal.h"
cbsl_errors create_streaming_buffers(cbsl_ctx* ctx)
{
ZSTD_CCtx* zstd_cctx = NULL;
ZSTD_DCtx* zstd_dctx = NULL;
uint64_t in_buffer_size;
uint64_t out_buffer_size;
void* in_buffer = NULL;
void* out_buffer = NULL;
void* line_buffer = NULL;
switch (ctx->mode)
{
case cbsl_load_mode:
zstd_dctx = ZSTD_createDCtx();
if (zstd_dctx == NULL) return cbsl_error;
in_buffer_size = ZSTD_DStreamInSize();
out_buffer_size = ZSTD_DStreamOutSize();
break;
case cbsl_store_mode:
zstd_cctx = ZSTD_createCCtx();
if (zstd_cctx == NULL) return cbsl_error;
in_buffer_size = ZSTD_CStreamInSize();
out_buffer_size = ZSTD_CStreamOutSize();
break;
default:
return cbsl_error;
}
in_buffer = (byte_t*)(malloc(in_buffer_size));
out_buffer = (byte_t*)(malloc(out_buffer_size));
line_buffer = (byte_t*)(malloc(CBSL_LINEBUF_SIZE));
if (in_buffer == NULL || out_buffer == NULL || line_buffer == NULL)
{
CBSL_SAFE_FREE_ZSTD_CCTX(zstd_cctx);
CBSL_SAFE_FREE_ZSTD_DCTX(zstd_dctx);
CBSL_SAFE_FREE(in_buffer);
CBSL_SAFE_FREE(out_buffer);
CBSL_SAFE_FREE(line_buffer);
return cbsl_error;
}
ctx->zstd_cctx = zstd_cctx;
ctx->zstd_dctx = zstd_dctx;
ctx->in_buffer = in_buffer;
ctx->out_buffer = out_buffer;
ctx->line_buffer = line_buffer;
ctx->in_buffer_size = in_buffer_size;
ctx->out_buffer_size = out_buffer_size;
ctx->in_buffer_pos = 0;
ctx->out_buffer_pos = 0;
ctx->in_buffer_used = 0;
ctx->out_buffer_used = 0;
ctx->line_buffer_used = 0;
ctx->line_buffer_read_from_zst = 0;
ctx->zstd_buf_end = 0;
ctx->zstd_file_end = 0;
return cbsl_success;
}

78
cachelab/cbsl/src/cbsl_internal.h Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CBSL_INTERNAL_HEADER_INCLUDED
#define CBSL_INTERNAL_HEADER_INCLUDED
#include <stdio.h>
#include <stdint.h>
#include <zstd.h>
#include <cbsl.h>
#define CBSL_LINEBUF_SIZE 8192
typedef unsigned char byte_t;
struct cbsl_ctx_
{
FILE* fp;
ZSTD_CCtx* zstd_cctx;
ZSTD_DCtx* zstd_dctx;
byte_t* in_buffer;
byte_t* out_buffer;
uint64_t in_buffer_size;
uint64_t out_buffer_size;
uint64_t in_buffer_pos;
uint64_t in_buffer_used;
uint64_t out_buffer_pos;
uint64_t out_buffer_used;
cbsl_mode mode;
// support readline
byte_t zstd_buf_end;
byte_t zstd_file_end;
byte_t* line_buffer;
uint64_t line_buffer_used;
uint64_t line_buffer_read_from_zst;
};
cbsl_errors create_streaming_buffers(cbsl_ctx* ctx);
#define MIN(x, y) (x) < (y) ? (x) : (y)
#ifdef CBSL_DEBUG
# include <assert.h>
# define CBSL_ASSERT(COND) { assert((COND)); }
# define CBSL_DEBUG_MESSAGE(...) { fprintf(stderr, __VA_ARGS__); }
#else
# define CBSL_ASSERT(COND) /* */
# define CBSL_DEBUG_MESSAGE(...) /* */
#endif
/* error check */
#define CBSL_CHECK_COND_MSG_IMPL(X,MSG,FINALIZE) { if (!(X)) { fprintf(stderr, "%s at %s l.%d: condition error %s; %s\n", __func__, __FILE__, __LINE__, (#X), (MSG)); FINALIZE; } }
#define CBSL_CHECK_COND(X) CBSL_CHECK_COND_MSG_IMPL((X), "", /* noreturn */)
#define CBSL_CHECK_COND_AND_RETURN(X,RETCODE) CBSL_CHECK_COND_MSG_IMPL((X), "", { return (RETCODE); })
#define CBSL_CHECK_COND_MSG_AND_RETURN(X,MSG,RETCODE) CBSL_CHECK_COND_MSG_IMPL((X), (MSG), { return (RETCODE); })
/* safe memory control */
#define CBSL_SAFE_FREE_RESOURCE(X,FREEFUNC) { if ((X) != NULL) { (FREEFUNC)((X)); (X) = NULL; } }
#define CBSL_SAFE_FCLOSE(X) CBSL_SAFE_FREE_RESOURCE((X), fclose);
#define CBSL_SAFE_FREE(X) CBSL_SAFE_FREE_RESOURCE((X), free)
#define CBSL_SAFE_FREE_ZSTD_CCTX(X) CBSL_SAFE_FREE_RESOURCE((X), ZSTD_freeCCtx)
#define CBSL_SAFE_FREE_ZSTD_DCTX(X) CBSL_SAFE_FREE_RESOURCE((X), ZSTD_freeDCtx)
#endif /* CBSL_INTERNAL_HEADER_INCLUDED */

94
cachelab/cbsl/src/file.c Normal file
View File

@@ -0,0 +1,94 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "./cbsl_internal.h"
static
cbsl_ctx* cbsl_open_safe_finalize(cbsl_ctx** ctx)
{
if (*ctx != NULL)
{
cbsl_close(*ctx);
}
return NULL;
}
cbsl_ctx* cbsl_open(cbsl_mode mode, char* path)
{
CBSL_CHECK_COND_AND_RETURN(path != NULL, NULL);
cbsl_ctx* ctx = (cbsl_ctx*)(malloc(sizeof(cbsl_ctx)));
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, NULL);
char const* fopen_mode;
switch (mode)
{
case cbsl_load_mode: fopen_mode = "rb"; break;
case cbsl_store_mode: fopen_mode = "wb"; break;
default: free(ctx); return NULL;
}
#ifdef _WIN32
errno_t err;
FILE* fp;
err = fopen_s(&fp, path, fopen_mode);
CBSL_CHECK_COND_AND_RETURN(err == 0, cbsl_open_safe_finalize(&ctx));
#else
FILE* fp = fopen(path, fopen_mode);
CBSL_CHECK_COND_AND_RETURN(fp != NULL, cbsl_open_safe_finalize(&ctx));
#endif
ctx->mode = mode;
ctx->fp = fp;
CBSL_CHECK_COND_AND_RETURN(create_streaming_buffers(ctx) == cbsl_success, cbsl_open_safe_finalize(&ctx));
/*
uint64_t file_version;
if (mode == cbsl_load_mode)
{
file_version = 0;
CBSL_CHECK_COND_AND_RETURN(cbsl_read(ctx, &file_version, sizeof(file_version)) == cbsl_success, cbsl_open_safe_finalize(&ctx));
CBSL_CHECK_COND_AND_RETURN(file_version == CBSL_VERSION, cbsl_open_safe_finalize(&ctx));
}
else if (mode == cbsl_store_mode)
{
file_version = CBSL_VERSION;
CBSL_CHECK_COND_AND_RETURN(cbsl_write(ctx, &file_version, sizeof(file_version)) == cbsl_success, cbsl_open_safe_finalize(&ctx));
}
*/
return ctx;
}
cbsl_errors cbsl_close(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_success);
if (ctx->mode == cbsl_store_mode)
{
cbsl_flush(ctx);
}
CBSL_SAFE_FREE_ZSTD_CCTX(ctx->zstd_cctx);
CBSL_SAFE_FREE_ZSTD_DCTX(ctx->zstd_dctx);
CBSL_SAFE_FREE(ctx->in_buffer);
CBSL_SAFE_FREE(ctx->out_buffer);
CBSL_SAFE_FREE(ctx->line_buffer);
CBSL_SAFE_FCLOSE(ctx->fp);
CBSL_SAFE_FREE(ctx);
return cbsl_success;
}

55
cachelab/cbsl/src/flush.c Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "./cbsl_internal.h"
static
cbsl_errors flush_serialize(cbsl_ctx* ctx)
{
if (ctx->in_buffer_used > 0)
{
ZSTD_inBuffer input = { ctx->in_buffer, ctx->in_buffer_used, 0 };
ZSTD_EndDirective mode = (ctx->in_buffer_used < ctx->in_buffer_size) ? ZSTD_e_end : ZSTD_e_continue;
do
{
ZSTD_outBuffer output = { ctx->out_buffer, ctx->out_buffer_size, 0 };
const size_t remaining = ZSTD_compressStream2(ctx->zstd_cctx, &output, &input, mode);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(remaining), ZSTD_getErrorName(remaining), cbsl_error);
const size_t written = fwrite(ctx->out_buffer, 1, output.pos, ctx->fp);
CBSL_CHECK_COND_AND_RETURN(written == output.pos, cbsl_error);
} while (input.pos < input.size);
ctx->in_buffer_used = 0;
}
fflush(ctx->fp);
return cbsl_success;
}
cbsl_errors cbsl_flush(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
if (ctx->mode == cbsl_store_mode)
{
return flush_serialize(ctx);
}
return cbsl_success;
}

471
cachelab/cbsl/src/fortran_bindings.f90.in Normal file
View File

@@ -0,0 +1,471 @@
!
! Copyright 2019 Yuta Hirokawa (University of Tsukuba)
!
! Licensed under the Apache License, Version 2.0 (the "License");
! you may not use this file except in compliance with the License.
! You may obtain a copy of the License at
!
! http://www.apache.org/licenses/LICENSE-2.0
!
! Unless required by applicable law or agreed to in writing, software
! distributed under the License is distributed on an "AS IS" BASIS,
! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
! See the License for the specific language governing permissions and
! limitations under the License.
!
module cbslf
use, intrinsic :: iso_c_binding
implicit none
type cbslf_context
integer(c_intptr_t) :: ctx
end type
public :: cbslf_context
public :: cbslf_store_mode
public :: cbslf_load_mode
public :: cbslf_unknown_mode
public :: cbslf_success
public :: cbslf_error
public :: cbslf_open
public :: cbslf_close
public :: cbslf_flush
public :: cbslf_write
public :: cbslf_read
public :: cbslf_record
public :: cbslf_record_heap
public :: cbslf_get_mode
public :: cbslf_set_compression_level
public :: cbslf_get_compression_level
#define PASTE(a,b) a##b
#define CONCAT(a,b) PASTE(a,b)
#define SET_ERRCODE(CODE) \
if (present(errcode)) errcode = CODE
#define SET_ERRCODE_COND(COND) \
if (present(errcode)) then \
__NL__ if (COND) then \
__NL__ errcode = cbslf_success \
__NL__ else \
__NL__ errcode = cbslf_error \
__NL__ end if \
__NL__ end if
#define FORTRAN_ARRAY_1 :
#define FORTRAN_ARRAY_2 :,:
#define FORTRAN_ARRAY_3 :,:,:
#define FORTRAN_ARRAY_4 :,:,:,:
#define FORTRAN_ARRAY_5 :,:,:,:,:
#define FORTRAN_ARRAY_6 :,:,:,:,:,:
#define FORTRAN_ARRAY_7 :,:,:,:,:,:,:
#define FORTRAN_ARRAY_VALUE_1 1
#define FORTRAN_ARRAY_VALUE_2 1,1
#define FORTRAN_ARRAY_VALUE_3 1,1,1
#define FORTRAN_ARRAY_VALUE_4 1,1,1,1
#define FORTRAN_ARRAY_VALUE_5 1,1,1,1,1
#define FORTRAN_ARRAY_VALUE_6 1,1,1,1,1,1
#define FORTRAN_ARRAY_VALUE_7 1,1,1,1,1,1,1
#define FORTRAN_ARRAY_SHAPE_1 lb(1):ub(1)
#define FORTRAN_ARRAY_SHAPE_2 lb(1):ub(1),lb(2):ub(2)
#define FORTRAN_ARRAY_SHAPE_3 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)
#define FORTRAN_ARRAY_SHAPE_4 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4)
#define FORTRAN_ARRAY_SHAPE_5 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4),lb(5):ub(5)
#define FORTRAN_ARRAY_SHAPE_6 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4),lb(5):ub(5),lb(6):ub(6)
#define FORTRAN_ARRAY_SHAPE_7 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4),lb(5):ub(5),lb(6):ub(6),lb(7):ub(7)
#define DECLARE_SCALAR_INTERFACE(SAVE_LOAD,ROUTINE_NAME) \
module procedure CONCAT(CONCAT(SAVE_LOAD,_scalar_),ROUTINE_NAME)
#define DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,DIMS) \
module procedure CONCAT(CONCAT(SAVE_LOAD,CONCAT(_array_,DIMS)),CONCAT(_,ROUTINE_NAME))
#define DECLARE_ARRAY_INTERFACE(SAVE_LOAD,ROUTINE_NAME) \
DECLARE_SCALAR_INTERFACE(SAVE_LOAD,ROUTINE_NAME) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,1) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,2) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,3) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,4) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,5) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,6) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,7) \
#define DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,DIMS) \
module procedure CONCAT(CONCAT(SAVE_LOAD,CONCAT(_array_heap_,DIMS)),CONCAT(_,ROUTINE_NAME))
#define DECLARE_ARRAY_INTERFACE_ALLOCATABLE(SAVE_LOAD,ROUTINE_NAME) \
DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,1) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,2) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,3) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,4) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,5) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,6) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,7)
#define IMPLEMENT_SCALAR_INTERFACE(ROUTINE_NAME,DATA_TYPE) \
subroutine CONCAT(write_scalar_,ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, target, intent(in) :: val \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(c_int64_t) :: psize \
__NL__ integer(4) :: ret \
__NL__ psize = c_sizeof(val) \
__NL__ ret = cbsl_write(ctx%ctx, loc(val), psize) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(read_scalar_,ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, target, intent(out) :: val \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(c_int64_t) :: psize \
__NL__ integer(4) :: ret \
__NL__ psize = c_sizeof(val) \
__NL__ ret = cbsl_read(ctx%ctx, loc(val), psize) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(record_scalar_,ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, target, intent(inout) :: val \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(c_int64_t) :: psize \
__NL__ integer(4) :: ret \
__NL__ psize = c_sizeof(val) \
__NL__ ret = cbsl_record(ctx%ctx, loc(val), psize) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine
#define IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,DIMS) \
__NL__ subroutine CONCAT(CONCAT(write_array_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, intent(in) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: lb(DIMS),ub(DIMS),nsize,i \
__NL__ integer(c_int64_t) :: bsize \
__NL__ integer(4) :: ret \
__NL__ nsize = size(val, kind=kind(nsize)) \
__NL__ do i=1,size(lb) \
__NL__ lb(i) = lbound(val, i, kind(lb(i))) \
__NL__ ub(i) = ubound(val, i, kind(ub(i))) \
__NL__ end do \
__NL__ call cbslf_write(ctx, nsize, ret) \
__NL__ bsize = size(lb) * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_write(ctx%ctx, loc(lb), bsize), ret) \
__NL__ ret = iand(cbsl_write(ctx%ctx, loc(ub), bsize), ret) \
__NL__ bsize = nsize * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_write(ctx%ctx, loc(val), bsize), ret) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(CONCAT(read_array_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, intent(out) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: lb(DIMS),ub(DIMS),nsize \
__NL__ integer(c_int64_t) :: bsize \
__NL__ integer(4) :: ret \
__NL__ ret = cbslf_error \
__NL__ if (size(val) > 0) then \
__NL__ call cbslf_read(ctx, nsize, ret) \
__NL__ if (ret == cbslf_success .and. nsize > 0) then \
__NL__ bsize = size(lb) * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(lb), bsize), ret) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(ub), bsize), ret) \
__NL__ bsize = nsize * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(val), bsize), ret) \
__NL__ else \
__NL__ ret = cbslf_error \
__NL__ end if \
__NL__ end if \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(CONCAT(record_array_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, allocatable, intent(inout) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: mode,ret \
__NL__ mode = cbslf_get_mode(ctx) \
__NL__ select case(mode) \
__NL__ case(cbslf_store_mode) \
__NL__ call cbslf_write(ctx, val, ret) \
__NL__ case(cbslf_load_mode) \
__NL__ call cbslf_read(ctx, val, ret) \
__NL__ case default \
__NL__ ret = cbslf_error \
__NL__ end select \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(CONCAT(record_array_heap_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, allocatable, intent(inout) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: lb(DIMS),ub(DIMS),nsize,mode \
__NL__ integer(4) :: ret \
__NL__ integer(c_int64_t) :: bsize \
__NL__ mode = cbslf_get_mode(ctx) \
__NL__ select case(mode) \
__NL__ case(cbslf_store_mode) \
__NL__ call cbslf_write(ctx, val, ret) \
__NL__ case(cbslf_load_mode) \
__NL__ if (allocated(val)) then \
__NL__ call cbslf_read(ctx, val, ret) \
__NL__ else \
__NL__ call cbslf_read(ctx, nsize, ret) \
__NL__ if (ret == cbslf_success .and. nsize > 0) then \
__NL__ bsize = size(lb) * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(lb), bsize), ret) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(ub), bsize), ret) \
__NL__ allocate(val(CONCAT(FORTRAN_ARRAY_SHAPE_,DIMS))) \
__NL__ bsize = nsize * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(val), bsize), ret) \
__NL__ else \
__NL__ ret = cbslf_error \
__NL__ end if \
__NL__ end if \
__NL__ case default \
__NL__ ret = cbslf_error \
__NL__ end select \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine
#define IMPLEMENT_ARRAY_INTERFACE(ROUTINE_NAME,DATA_TYPE) \
IMPLEMENT_SCALAR_INTERFACE(ROUTINE_NAME,DATA_TYPE) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,1) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,2) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,3) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,4) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,5) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,6) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,7)
!======
private
integer(8), parameter :: CBSL_VERSION = @cbsl_VERSION_MAJOR@@cbsl_VERSION_MINOR@@cbsl_VERSION_PATCH@
character(*), parameter :: CBSL_VERSTION_STRING = '@cbsl_VERSION@'
integer(4), parameter :: CBSL_MAJOR_VERSION = @cbsl_VERSION_MAJOR@
integer(4), parameter :: CBSL_MINOR_VERSION = @cbsl_VERSION_MINOR@
integer(4), parameter :: CBSL_PATCH_VERSION = @cbsl_VERSION_PATCH@
integer(4), parameter :: cbslf_load_mode = 1
integer(4), parameter :: cbslf_store_mode = 2
integer(4), parameter :: cbslf_unknown_mode = -1
integer(4), parameter :: cbslf_success = 0
integer(4), parameter :: cbslf_error = -1
! C function interfaces
interface
function cbsl_open(open_mode, path) result(ctx) &
bind(C,name='cbsl_open')
use, intrinsic :: iso_c_binding
integer(c_int),value, intent(in) :: open_mode
character(kind=c_char),intent(in) :: path(*)
integer(c_intptr_t) :: ctx
end function
function cbsl_close(ctx) result(retcode) &
bind(C,name='cbsl_close')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: retcode
end function
function cbsl_flush(ctx) result(retcode) &
bind(C,name='cbsl_flush')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: retcode
end function
function cbsl_get_mode(ctx) result(retcode) &
bind(C,name='cbsl_get_mode')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: retcode
end function
function cbsl_write(ctx, pdata, psize) result(retcode) &
bind(C,name='cbsl_write')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_intptr_t),value,intent(in) :: pdata
integer(c_int64_t),value,intent(in) :: psize
integer(c_int) :: retcode
end function
function cbsl_read(ctx, pdata, psize) result(retcode) &
bind(C,name='cbsl_read')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_intptr_t),value,intent(in) :: pdata
integer(c_int64_t),value,intent(in) :: psize
integer(c_int) :: retcode
end function
function cbsl_record(ctx, pdata, psize) result(retcode) &
bind(C,name='cbsl_record')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_intptr_t),value,intent(in) :: pdata
integer(c_int64_t),value,intent(in) :: psize
integer(c_int) :: retcode
end function
! cbsl_record_heap() not binds because the function uses malloc()
function cbsl_set_compression_level(ctx,clevel) result(retcode) &
bind(C,name='cbsl_set_compression_level')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int),value,intent(in) :: clevel
integer(c_int) :: retcode
end function
function cbsl_get_compression_level(ctx) result(clevel) &
bind(C,name='cbsl_get_compression_level')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: clevel
end function
end interface
! Generic routines
interface cbslf_write
DECLARE_SCALAR_INTERFACE(write,character)
DECLARE_SCALAR_INTERFACE(write,logical)
DECLARE_ARRAY_INTERFACE(write,integer4)
DECLARE_ARRAY_INTERFACE(write,integer8)
DECLARE_ARRAY_INTERFACE(write,sreal)
DECLARE_ARRAY_INTERFACE(write,dreal)
DECLARE_ARRAY_INTERFACE(write,ccomplex)
DECLARE_ARRAY_INTERFACE(write,zcomplex)
end interface
interface cbslf_read
DECLARE_SCALAR_INTERFACE(read,character)
DECLARE_SCALAR_INTERFACE(read,logical)
DECLARE_ARRAY_INTERFACE(read,integer4)
DECLARE_ARRAY_INTERFACE(read,integer8)
DECLARE_ARRAY_INTERFACE(read,sreal)
DECLARE_ARRAY_INTERFACE(read,dreal)
DECLARE_ARRAY_INTERFACE(read,ccomplex)
DECLARE_ARRAY_INTERFACE(read,zcomplex)
end interface
interface cbslf_record
DECLARE_SCALAR_INTERFACE(record,character)
DECLARE_SCALAR_INTERFACE(record,logical)
DECLARE_ARRAY_INTERFACE(record,integer4)
DECLARE_ARRAY_INTERFACE(record,integer8)
DECLARE_ARRAY_INTERFACE(record,sreal)
DECLARE_ARRAY_INTERFACE(record,dreal)
DECLARE_ARRAY_INTERFACE(record,ccomplex)
DECLARE_ARRAY_INTERFACE(record,zcomplex)
end interface
interface cbslf_record_heap
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,integer4)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,integer8)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,sreal)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,dreal)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,ccomplex)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,zcomplex)
end interface
contains
IMPLEMENT_SCALAR_INTERFACE(character,character(*,kind=c_char))
IMPLEMENT_SCALAR_INTERFACE(logical,logical(c_bool))
IMPLEMENT_ARRAY_INTERFACE(integer4,integer(c_int32_t))
IMPLEMENT_ARRAY_INTERFACE(integer8,integer(c_int64_t))
IMPLEMENT_ARRAY_INTERFACE(sreal,real(c_float))
IMPLEMENT_ARRAY_INTERFACE(dreal,real(c_double))
IMPLEMENT_ARRAY_INTERFACE(ccomplex,complex(c_float_complex))
IMPLEMENT_ARRAY_INTERFACE(zcomplex,complex(c_double_complex))
function cbslf_open(open_mode, path, errcode) result(ctx)
implicit none
integer(4), intent(in) :: open_mode
character(*), intent(in) :: path
integer(4), intent(out), optional :: errcode
type(cbslf_context) :: ctx
character(1,c_char) :: cpath(len_trim(path)+1)
integer(4) :: n,i
n = len_trim(path, kind=4)
do i=1,n
cpath(i) = path(i:i)
end do
cpath(n+1) = c_null_char
ctx%ctx = cbsl_open(open_mode, cpath)
SET_ERRCODE_COND(ctx%ctx /= 0) ! is not null pointer
end function
subroutine cbslf_close(ctx, errcode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: ret
ret = cbsl_close(ctx%ctx)
SET_ERRCODE(ret)
end subroutine
subroutine cbslf_flush(ctx, errcode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: ret
ret = cbsl_flush(ctx%ctx)
SET_ERRCODE(ret)
end subroutine
function cbslf_get_mode(ctx, errcode) result(open_mode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: open_mode
open_mode = cbsl_get_mode(ctx%ctx)
SET_ERRCODE_COND(open_mode == cbslf_unknown_mode)
end function
subroutine cbslf_set_compression_level(ctx, clevel, errcode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(in) :: clevel
integer(4), intent(out), optional :: errcode
integer(4) :: ret
ret = cbsl_set_compression_level(ctx%ctx, clevel)
SET_ERRCODE(ret)
end subroutine
function cbslf_get_compression_level(ctx, errcode) result(clevel)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: clevel
clevel = cbsl_get_compression_level(ctx%ctx)
SET_ERRCODE_COND(clevel > 0)
end function
end module

31
cachelab/cbsl/src/gen_fortran_bindings.cmake Normal file
View File

@@ -0,0 +1,31 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
cmake_minimum_required(VERSION 3.3)
if (${CMAKE_ARGC} LESS 4)
message(FATAL_ERROR "Usage ${CMAKE_ARGV2} <input file> <output file>")
endif ()
set(INPUT_FILE ${CMAKE_ARGV3})
if (CMAKE_ARGV4)
set(OUTPUT_FILE ${CMAKE_ARGV4})
else ()
set(OUTPUT_FILE ${CMAKE_ARGV3})
endif ()
file(READ ${INPUT_FILE} FBIND_SOURCE_TEXT)
string(REGEX REPLACE "__NL__" "\n" FBIND_GEN_TEXT ${FBIND_SOURCE_TEXT})
file(WRITE ${OUTPUT_FILE} ${FBIND_GEN_TEXT})

208
cachelab/cbsl/src/read.c Normal file
View File

@@ -0,0 +1,208 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <zstd.h>
#include "./cbsl_internal.h"
static
cbsl_errors streaming_read_immediate(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
/* previous all decompressed data loaded */
CBSL_CHECK_COND_AND_RETURN(ctx->out_buffer_pos >= ctx->out_buffer_used, cbsl_error);
ZSTD_outBuffer output = { data, size, 0 }; /* decompressed data stores directly */
while (output.pos < output.size)
{
/* previous read data from file */
if (ctx->in_buffer_pos < ctx->in_buffer_used)
{
/* input buffer has remained data from the previous decompression */
memmove(ctx->in_buffer, ctx->in_buffer + ctx->in_buffer_pos, ctx->in_buffer_used - ctx->in_buffer_pos);
ctx->in_buffer_used -= ctx->in_buffer_pos;
}
else
{
ctx->in_buffer_used = 0ULL;
}
ctx->in_buffer_pos = 0ULL;
ctx->in_buffer_used += fread(ctx->in_buffer + ctx->in_buffer_used, 1, ctx->in_buffer_size - ctx->in_buffer_used, ctx->fp);
ZSTD_inBuffer input = { ctx->in_buffer, ctx->in_buffer_used, ctx->in_buffer_pos };
while (input.pos < input.size && output.pos < output.size)
{
const size_t ret = ZSTD_decompressStream(ctx->zstd_dctx, &output, &input);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(ret), ZSTD_getErrorName(ret), cbsl_error);
}
ctx->in_buffer_used = input.size;
ctx->in_buffer_pos = input.pos;
}
CBSL_ASSERT(output.pos == output.size);
ctx->out_buffer_pos = output.pos;
ctx->out_buffer_used = output.size;
return cbsl_success;
}
static
uint64_t read_from_buffer(cbsl_ctx* ctx, byte_t* data, uint64_t size)
{
uint64_t read = 0;
if (ctx->out_buffer_pos < ctx->out_buffer_used)
{
const uint64_t read_size = MIN(ctx->out_buffer_used - ctx->out_buffer_pos, size);
memcpy(data, ctx->out_buffer + ctx->out_buffer_pos, read_size);
ctx->out_buffer_pos += read_size;
read = read_size;
}
return read;
}
static
cbsl_errors streaming_read_buffered(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
size_t zstd_ret = -1;
/* memcpy from the previous decompression data */
uint64_t read = read_from_buffer(ctx, data, size);
byte_t* pdata = (byte_t*)(data);
/* streaming decompression and memcpy */
while (read < size)
{
if (ctx->in_buffer_pos < ctx->in_buffer_used)
{
/* input buffer has remained data from the previous decompression */
memmove(ctx->in_buffer, ctx->in_buffer + ctx->in_buffer_pos, ctx->in_buffer_used - ctx->in_buffer_pos);
ctx->in_buffer_used -= ctx->in_buffer_pos;
}
else
{
/* input buffer is already consumed by the previous decompression */
ctx->in_buffer_used = 0;
}
ctx->in_buffer_pos = 0ULL;
ctx->in_buffer_used += fread(ctx->in_buffer + ctx->in_buffer_used, 1, ctx->in_buffer_size - ctx->in_buffer_used, ctx->fp);
if (ctx->in_buffer_used == 0) // 从文件中，读取不到数据了，无法解压
break;
ZSTD_inBuffer input = { ctx->in_buffer, ctx->in_buffer_used, ctx->in_buffer_pos };
while (input.pos < input.size && read < size)
{
ZSTD_outBuffer output = { ctx->out_buffer, ctx->out_buffer_size, 0 };
zstd_ret = ZSTD_decompressStream(ctx->zstd_dctx, &output, &input);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(zstd_ret), ZSTD_getErrorName(zstd_ret), cbsl_error);
ctx->out_buffer_pos = 0ULL;
ctx->out_buffer_used = output.pos;
read += read_from_buffer(ctx, pdata + read, size - read);
}
ctx->in_buffer_pos = input.pos;
ctx->in_buffer_used = input.size;
if (zstd_ret == 0)
{
// zstd stream end
ctx->zstd_file_end = 1;
break;
}
}
if (ctx->zstd_file_end == 1 && ctx->out_buffer_pos == ctx->out_buffer_used)
ctx->zstd_buf_end = 1;
CBSL_ASSERT(read == size || ctx->zstd_end == 1);
ctx->line_buffer_read_from_zst = read;
return cbsl_success;
}
cbsl_errors cbsl_read(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_load_mode, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_DEBUG_MESSAGE("read: %lu bytes\n", size);
if (size > ctx->out_buffer_size)
{
/* data size is larger than compression buffer */
return streaming_read_immediate(ctx, data, size);
}
else
{
return streaming_read_buffered(ctx, data, size);
}
}
// readline from stream
cbsl_errors cbsl_readline(cbsl_ctx* ctx, char* data, uint64_t size)
{
cbsl_errors ret = cbsl_success;
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_load_mode, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_DEBUG_MESSAGE("%s: %lu bytes\n", __func__, size);
uint64_t line_buffer_space = 0;
line_buffer_space = CBSL_LINEBUF_SIZE - ctx->line_buffer_used;
if (ctx->zstd_buf_end == 0)
{
if (line_buffer_space > ctx->out_buffer_size)
{
/* line_buffer size is larger than compression buffer */
ret = streaming_read_immediate(ctx, ctx->line_buffer + ctx->line_buffer_used, line_buffer_space);
}
else
{
ret = streaming_read_buffered(ctx, ctx->line_buffer + ctx->line_buffer_used, line_buffer_space);
}
ctx->line_buffer_used += ctx->line_buffer_read_from_zst;
}
uint64_t i;
char* ptr;
ptr = (char*)ctx->line_buffer;
data[0] = '\0';
for (i = 0; i < ctx->line_buffer_used; i++)
{
if (*ptr == '\n' || *ptr == '\r')
{
CBSL_CHECK_COND_AND_RETURN(size >= i - 1, cbsl_error);
memcpy(data, ctx->line_buffer, i + 1);
memmove(ctx->line_buffer, ctx->line_buffer + i + 1, ctx->line_buffer_used - i - 1);
ctx->line_buffer_used -= i + 1;
data[i + 1] = '\0';
break;
}
ptr++;
}
if (ctx->line_buffer_used == 0)
return cbsl_end;
else
return ret;
}

70
cachelab/cbsl/src/record.c Normal file
View File

@@ -0,0 +1,70 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdint.h>
#include <stdlib.h>
#include "./cbsl_internal.h"
cbsl_errors cbsl_record(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
switch (ctx->mode)
{
case cbsl_load_mode: return cbsl_read(ctx, data, size);
case cbsl_store_mode: return cbsl_write(ctx, data, size);
default: return cbsl_error;
}
}
cbsl_errors cbsl_record_heap(cbsl_ctx* ctx, void** data, uint64_t* size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(size != NULL, cbsl_error);
switch (ctx->mode)
{
case cbsl_load_mode:
{
uint64_t rsize;
void* rdata = *data;
CBSL_CHECK_COND_AND_RETURN(cbsl_read(ctx, &rsize, sizeof(rsize)) == cbsl_success, cbsl_error);
if (rdata == NULL)
{
CBSL_CHECK_COND_AND_RETURN((rdata = malloc(rsize)) != NULL, cbsl_error);
}
else
{
CBSL_CHECK_COND_AND_RETURN(rsize == *size, cbsl_error);
}
CBSL_CHECK_COND_AND_RETURN(cbsl_read(ctx, rdata, rsize) == cbsl_success, cbsl_error);
*size = rsize;
*data = rdata;
}
break;
case cbsl_store_mode:
{
CBSL_CHECK_COND_AND_RETURN(cbsl_write(ctx, size, sizeof(*size)) == cbsl_success, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(cbsl_write(ctx, *data, *size) == cbsl_success, cbsl_error);
}
break;
default:
return cbsl_error;
}
return cbsl_success;
}

49
cachelab/cbsl/src/utils.c Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "./cbsl_internal.h"
/* FIXME: implicit declaration??? */
extern ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
cbsl_mode cbsl_get_mode(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_unknown_mode);
return ctx->mode;
}
cbsl_errors cbsl_set_compression_level(cbsl_ctx* ctx, int clevel)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_store_mode, cbsl_error);
const size_t ret = ZSTD_CCtx_setParameter(ctx->zstd_cctx, ZSTD_c_compressionLevel, clevel);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(ret), ZSTD_getErrorName(ret), cbsl_error);
return cbsl_success;
}
int cbsl_get_compression_level(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_store_mode, cbsl_error);
int clevel;
const size_t ret = ZSTD_CCtx_getParameter(ctx->zstd_cctx, ZSTD_c_compressionLevel, &clevel);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(ret), ZSTD_getErrorName(ret), cbsl_error);
return clevel;
}

101
cachelab/cbsl/src/write.c Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include "./cbsl_internal.h"
static
cbsl_errors streaming_write_flush(cbsl_ctx* ctx, uint64_t size)
{
if (ctx->in_buffer_used > 0 && ctx->in_buffer_used + size > ctx->in_buffer_size)
{
CBSL_CHECK_COND_AND_RETURN(cbsl_flush(ctx) == cbsl_success, cbsl_error);
}
return cbsl_success;
}
static
cbsl_errors streaming_write_immediate(cbsl_ctx* ctx, const void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
CBSL_CHECK_COND_AND_RETURN(streaming_write_flush(ctx, size) == cbsl_success, cbsl_error);
uint64_t written = 0;
const byte_t* pdata = (const byte_t*)(data);
while (written < size)
{
/* compressing data copies to internal input buffer */
const uint64_t write_size = MIN(size - written, ctx->in_buffer_size);
memcpy(ctx->in_buffer, pdata + written, write_size);
written += write_size;
ZSTD_inBuffer input = { ctx->in_buffer, write_size, 0 };
ZSTD_EndDirective mode = (size - written < ctx->in_buffer_size) ? ZSTD_e_end : ZSTD_e_continue;
do
{
ZSTD_outBuffer output = { ctx->out_buffer, ctx->out_buffer_size, 0 };
const size_t remaining = ZSTD_compressStream2(ctx->zstd_cctx, &output, &input, mode);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(remaining), ZSTD_getErrorName(remaining), cbsl_error);
CBSL_CHECK_COND_AND_RETURN(fwrite(ctx->out_buffer, 1, output.pos, ctx->fp) == output.pos, cbsl_error);
} while (input.pos < input.size);
CBSL_ASSERT(input.pos == input.size);
}
CBSL_ASSERT(written == size);
return cbsl_success;
}
static
cbsl_errors streaming_write_buffered(cbsl_ctx* ctx, const void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
CBSL_CHECK_COND_AND_RETURN(streaming_write_flush(ctx, size) == cbsl_success, cbsl_error);
uint64_t written = 0;
const byte_t* pdata = (const byte_t*)(data);
do
{
const uint64_t write_size = MIN(size - written, ctx->in_buffer_size);
memcpy(ctx->in_buffer, pdata + written, write_size);
ctx->in_buffer_used = write_size;
CBSL_CHECK_COND_AND_RETURN(cbsl_flush(ctx) == cbsl_success, cbsl_error);
written += write_size;
} while (written < size);
CBSL_ASSERT(written == size);
return cbsl_success;
}
cbsl_errors cbsl_write(cbsl_ctx* ctx, const void* data, uint64_t size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_store_mode, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_DEBUG_MESSAGE("write: %lu bytes\n", size);
if (size > ctx->in_buffer_size)
{
/* data size is larger than compression buffer */
return streaming_write_immediate(ctx, data, size);
}
else
{
return streaming_write_buffered(ctx, data, size);
}
}

106
cachelab/cbsl/tests/CMakeLists.txt Normal file
View File

@@ -0,0 +1,106 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
link_libraries(${CBSL_LIB} ${ZSTD_LIB})
add_executable(scalar_data scalar_data.c)
add_executable(single_array single_array.c)
add_executable(multi_array multi_array.c)
add_executable(utility utility.c)
add_executable(split_compression split_compression.c)
add_executable(variable_size_array variable_size_array.c)
add_executable(various_size_array various_size_array.c)
add_executable(cbsl_record cbsl_record.c)
add_executable(cbsl_record_array cbsl_record_array.c)
add_executable(fortran_bindings fortran_bindings.f90)
target_link_libraries(fortran_bindings ${CBSL_FLIB} ${CBSL_LIB} ${ZSTD_LIB})
add_test(NAME utility COMMAND utility)
#
# very short data compression/decompression
#
add_test(NAME scalar_data_compression COMMAND scalar_data -c)
add_test(NAME scalar_data_decompression COMMAND scalar_data -d)
set_tests_properties(scalar_data_decompression PROPERTIES DEPENDS scalar_data_compression)
#
# boundary checking with single array
#
function(add_simple_test_size_boundary DATA_SIZE)
math(EXPR DATA_SIZE_B "${DATA_SIZE} - 1")
add_test(NAME single_array_${DATA_SIZE_B} COMMAND single_array ${DATA_SIZE_B})
add_test(NAME single_array_${DATA_SIZE} COMMAND single_array ${DATA_SIZE})
math(EXPR DATA_SIZE_B "${DATA_SIZE} + 1")
add_test(NAME single_array_${DATA_SIZE_B} COMMAND single_array ${DATA_SIZE_B})
endfunction()
foreach(count RANGE 10 20)
math(EXPR DATA_SIZE "1 << ${count}")
add_simple_test_size_boundary(${DATA_SIZE})
endforeach()
#
# boundary checking with multiple arrays
#
function(add_multiple_test_size NUM_VARS)
foreach(count RANGE 10 20)
math(EXPR DATA_SIZE "1 << ${count}")
add_test(NAME multi_array_${DATA_SIZE}_by_${NUM_VARS} COMMAND multi_array ${NUM_VARS} ${DATA_SIZE})
endforeach()
endfunction()
add_multiple_test_size(2)
add_multiple_test_size(3)
add_multiple_test_size(11)
#
# test compression/decompression form
#
add_test(NAME variable_size_array COMMAND variable_size_array)
math(EXPR DATA_SIZE "1 << 17")
add_test(NAME buffered_read_write COMMAND variable_size_array ${DATA_SIZE})
math(EXPR DATA_SIZE "(1 << 17) + 1")
add_test(NAME immediate_read_write COMMAND variable_size_array ${DATA_SIZE})
add_test(NAME various_size_array COMMAND various_size_array)
add_test(NAME cbsl_record COMMAND cbsl_record)
add_test(NAME cbsl_record_array COMMAND cbsl_record_array)
#
# large array compression/decompression by splitting small block
#
math(EXPR BLOCK_SIZE "1 << 18") # 256 KiB
math(EXPR TOTAL_SIZE "1 << 28") # 256 MiB
add_test(NAME split_compression_256KiB_256MiB COMMAND split_compression ${BLOCK_SIZE} ${TOTAL_SIZE})
math(EXPR BLOCK_SIZE "(1 << 18) - (1 << 16)") # 192 KiB
math(EXPR TOTAL_SIZE "1 << 28") # 256 MiB
add_test(NAME split_compression_192KiB_256MiB COMMAND split_compression ${BLOCK_SIZE} ${TOTAL_SIZE})
#
# Fortran bindings API
#
add_test(NAME fortran_bindings COMMAND fortran_bindings)

64
cachelab/cbsl/tests/cbsl_record.c Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
#define CHECK_BINARY(X,Y) {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
static const int data0 = 43;
static const double data1 = 3.14159265;
static const int data2 = 14142;
extern void record(cbsl_mode, int*, double*, int*);
char cname[128];
int main(int argc, char** argv)
{
sprintf(cname, "check_record_scalar.zst");
int c0 = data0;
double c1 = data1;
int c2 = data2;
record(cbsl_store_mode, &c0, &c1, &c2);
c0 = c2 = 0;
c1 = 0;
record(cbsl_load_mode, &c0, &c1, &c2);
CHECK_BINARY(c0, data0);
CHECK_BINARY(c1, data1);
CHECK_BINARY(c2, data2);
return 0;
}
void record(cbsl_mode mode, int* c0, double* c1, int* c2)
{
cbsl_ctx* ctx = cbsl_open(mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_record(ctx, c0, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_record(ctx, c1, sizeof(double)));
CBSL_ERROR_CHECK(cbsl_record(ctx, c2, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}

96
cachelab/cbsl/tests/cbsl_record_array.c Normal file
View File

@@ -0,0 +1,96 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
#define CHECK(X) {if (!(X)) { fprintf(stderr, "error: check %s\n", (#X)); exit(1); }}
#define CHECK_BINARY(X,Y) {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
typedef unsigned char byte_t;
extern void rand_byte_t(uint64_t data_size, byte_t* a);
extern void record(cbsl_mode, uint64_t* size, void** data);
char cname[128];
int main(int argc, char** argv)
{
sprintf(cname, "check_record_scalar.zst");
uint64_t size = 1024 * 1024 * sizeof(byte_t);
byte_t* data = (byte_t*)(malloc(size));
rand_byte_t(size, data);
record(cbsl_store_mode, &size, (void**) &data);
uint64_t rsize = 0;
byte_t* rdata = NULL;
record(cbsl_load_mode, &rsize, (void**) &rdata);
CHECK(size == rsize);
CHECK(rdata != NULL);
for (uint64_t i = 0; i < (size/sizeof(byte_t)); ++i)
{
if (data[i] != rdata[i])
{
fprintf(stderr, "1: mismatch!\n");
exit(1);
}
else
{
rdata[i] = 0;
}
}
CHECK(size == rsize);
CHECK(rdata != NULL);
record(cbsl_load_mode, &rsize, (void**) &rdata);
for (uint64_t i = 0; i < (size/sizeof(byte_t)); ++i)
{
if (data[i] != rdata[i])
{
fprintf(stderr, "2: mismatch!\n");
exit(1);
}
}
free(data);
free(rdata);
return 0;
}
void record(cbsl_mode mode, uint64_t* size, void** data)
{
cbsl_ctx* ctx = cbsl_open(mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_record_heap(ctx, data, size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void rand_byte_t(uint64_t data_size, byte_t* a)
{
for(uint64_t i = 0; i < data_size; ++i)
a[i] = rand() % 255;
}

159
cachelab/cbsl/tests/fortran_bindings.f90 Normal file
View File

@@ -0,0 +1,159 @@
!
! Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
!
! Licensed under the Apache License, Version 2.0 (the "License");
! you may not use this file except in compliance with the License.
! You may obtain a copy of the License at
!
! http://www.apache.org/licenses/LICENSE-2.0
!
! Unless required by applicable law or agreed to in writing, software
! distributed under the License is distributed on an "AS IS" BASIS,
! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
! See the License for the specific language governing permissions and
! limitations under the License.
!
program test_fortran_bindings
use cbslf
integer, parameter :: array_size = 100000
real :: source_stack_array(array_size), dest_stack_array(array_size)
real, allocatable :: source_heap_array(:), dest_heap_array(:), dest_heap_array2(:)
character(128) :: source_string, dest_string
character(*), parameter :: cname = 'data_fortran.zst'
allocate(source_heap_array(array_size))
allocate(dest_heap_array(array_size))
call random_number(source_stack_array)
call random_number(source_heap_array)
source_string = 'abcdefgABCDEFG%'
dest_string = ''
print *, 'serialize...'
call serialize
print *, 'deserialize...'
call deserialize
print *, 'compare...'
if (.not. compare_real_array(array_size, source_stack_array, dest_stack_array)) then
print *, 'fail: compare stack array'
stop 1
end if
if (.not. compare_real_array(array_size, source_heap_array, dest_heap_array)) then
print *, 'fail: compare heap array'
stop 1
end if
if (.not. compare_real_array(array_size, source_heap_array, dest_heap_array2)) then
print *, 'fail: compare heap array'
stop 1
end if
if (trim(source_string) /= trim(dest_string)) then
print *, 'fail: compare string'
stop 1
end if
contains
subroutine serialize
implicit none
type(cbslf_context) :: ctx
integer(4) :: errcode
ctx = cbslf_open(cbslf_store_mode, cname, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_open(store)'
stop 1
end if
call cbslf_set_compression_level(ctx, 10, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_set_compression_level'
stop 1
end if
call cbslf_write(ctx, source_stack_array, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_write(stack array)'
stop 1
end if
call cbslf_write(ctx, source_heap_array, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_write(heap array)'
stop 1
end if
call cbslf_write(ctx, source_heap_array, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_write(heap array2)'
stop 1
end if
call cbslf_write(ctx, source_string, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_write(string)'
stop 1
end if
call cbslf_close(ctx)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_close'
stop 1
end if
end subroutine
subroutine deserialize
implicit none
type(cbslf_context) :: ctx
integer(4) :: errcode
ctx = cbslf_open(cbslf_load_mode, cname, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_open(load)'
stop 1
end if
call cbslf_read(ctx, dest_stack_array, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_read(stack array)'
stop 1
end if
call cbslf_read(ctx, dest_heap_array, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_read(heap array)'
stop 1
end if
call cbslf_record_heap(ctx, dest_heap_array2, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_record_heap(heap array2)'
stop 1
end if
call cbslf_read(ctx, dest_string, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_read(string)'
stop 1
end if
call cbslf_close(ctx, errcode)
if (errcode /= cbslf_success) then
print *, 'fail: cbslf_close'
stop 1
end if
end subroutine
function compare_real_array(n, a, b) result(ret)
implicit none
integer, intent(in) :: n
real, intent(in) :: a(n), b(n)
logical :: ret
integer :: i
do i=1,n
ret = (abs(a(i) - b(i)) <= epsilon(a(i)))
end do
end function
end program

150
cachelab/cbsl/tests/multi_array.c Normal file
View File

@@ -0,0 +1,150 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <math.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
typedef unsigned char byte_t;
extern void compress(uint64_t num_vars, uint64_t data_size, byte_t** a);
extern void raw_write(uint64_t num_vars, uint64_t data_size, byte_t** a);
extern void decompress(uint64_t num_vars, uint64_t data_size, byte_t** a);
extern void raw_read(uint64_t num_vars, uint64_t data_size, byte_t** a);
extern void rand_byte_t(uint64_t data_size, byte_t* a);
char cname[128], sname[128];
int main(int argc, char** argv)
{
if (argc <= 2)
exit(1);
uint64_t data_size, num_vars;
sscanf(argv[1], "%lu", &num_vars);
sscanf(argv[2], "%lu", &data_size);
data_size = data_size / num_vars;
printf("variables = %lu\n", num_vars);
printf("data size = %lf [MiB] * %lu\n", (double)(data_size)/pow(2,20), num_vars);
sprintf(cname, "multiple_compressed_%lu.zst", data_size * num_vars);
sprintf(sname, "multiple_raw_%lu.dat", data_size);
srand((unsigned int)(time(NULL)));
byte_t** a = malloc(sizeof(byte_t*) * num_vars);
byte_t** b = malloc(sizeof(byte_t*) * num_vars);
byte_t** c = malloc(sizeof(byte_t*) * num_vars);
for(uint64_t i = 0; i < num_vars; ++i)
{
a[i] = (byte_t*)(malloc(data_size));
rand_byte_t(data_size / sizeof(byte_t), a[i]);
b[i] = (byte_t*)(malloc(data_size));
c[i] = (byte_t*)(malloc(data_size));
}
compress(num_vars, data_size, a);
raw_write(num_vars, data_size, a);
decompress(num_vars, data_size, b);
raw_read(num_vars, data_size, c);
for(uint64_t i = 0; i < num_vars; ++i)
{
for(uint64_t j = 0; j < data_size; ++j)
{
if (b[i][j] != c[i][j])
{
fprintf(stderr, "mismatch!\n");
exit(1);
}
}
}
for(uint64_t i = 0; i < num_vars; ++i)
{
free(a[i]);
free(b[i]);
free(c[i]);
}
free(a);
free(b);
free(c);
return 0;
}
void compress(uint64_t num_vars, uint64_t data_size, byte_t** a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
for(uint64_t i = 0; i < num_vars; ++i)
CBSL_ERROR_CHECK(cbsl_write(ctx, a[i], data_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_write(uint64_t num_vars, uint64_t data_size, byte_t** a)
{
FILE* fp = fopen(sname, "wb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen\n");
exit(1);
}
for(int i = 0; i < num_vars; ++i)
fwrite(a[i], 1, data_size, fp);
fclose(fp);
}
void decompress(uint64_t num_vars, uint64_t data_size, byte_t** a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
for(uint64_t i = 0; i < num_vars; ++i)
CBSL_ERROR_CHECK(cbsl_read(ctx, a[i], data_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_read(uint64_t num_vars, uint64_t data_size, byte_t** a)
{
FILE* fp = fopen(sname, "rb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen\n");
exit(1);
}
for(uint64_t i = 0; i < num_vars; ++i)
fread(a[i], 1, data_size, fp);
fclose(fp);
}
void rand_byte_t(uint64_t data_size, byte_t* a)
{
for(uint64_t i = 0; i < data_size; ++i)
a[i] = rand() % 255;
}

111
cachelab/cbsl/tests/scalar_data.c Normal file
View File

@@ -0,0 +1,111 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
#define CHECK_BINARY(X,Y) {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
static const int data0 = 43;
static const double data1 = 3.14159265;
static const int data2 = 14142;
extern void compress();
extern void decompress();
char cname[128], sname[128];
int main(int argc, char** argv)
{
if (argc < 2)
return 1;
sprintf(cname, "scalar_compressed.zst");
sprintf(sname, "scalar_raw.dat");
if (strcmp(argv[1],"-c") == 0)
compress();
else if (strcmp(argv[1],"-d") == 0)
decompress();
else
return 1;
return 0;
}
void compress()
{
const int c0 = data0;
const double c1 = data1;
const int c2 = data2;
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_write(ctx, &c0, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_write(ctx, &c1, sizeof(double)));
CBSL_ERROR_CHECK(cbsl_write(ctx, &c2, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_close(ctx));
FILE* fp = fopen(sname, "wb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen\n");
exit(1);
}
fwrite(&c0, sizeof(int), 1, fp);
fwrite(&c1, sizeof(double), 1, fp);
fwrite(&c2, sizeof(int), 1, fp);
fclose(fp);
}
void decompress()
{
int d0, r0;
double d1, r1;
int d2, r2;
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_read(ctx, &d0, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_read(ctx, &d1, sizeof(double)));
CBSL_ERROR_CHECK(cbsl_read(ctx, &d2, sizeof(int)));
CBSL_ERROR_CHECK(cbsl_close(ctx));
FILE* fp = fopen(sname, "rb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen\n");
exit(1);
}
fread(&r0, sizeof(int), 1, fp);
fread(&r1, sizeof(double), 1, fp);
fread(&r2, sizeof(int), 1, fp);
fclose(fp);
CHECK_BINARY(d0, r0); CHECK_BINARY(d0, data0);
CHECK_BINARY(d1, r1); CHECK_BINARY(d1, data1);
CHECK_BINARY(d2, r2); CHECK_BINARY(d2, data2);
}

129
cachelab/cbsl/tests/single_array.c Normal file
View File

@@ -0,0 +1,129 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <math.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
typedef unsigned char byte_t;
extern void compress(uint64_t data_size, const byte_t* a);
extern void raw_write(uint64_t data_size, const byte_t* a);
extern void decompress(uint64_t data_size, byte_t* a);
extern void raw_read(uint64_t data_size, byte_t* a);
extern void rand_byte_t(uint64_t data_size, byte_t* a);
char cname[128], sname[128];
int main(int argc, char** argv)
{
if (argc < 2)
exit(1);
int data_size;
sscanf(argv[1], "%d", &data_size);
printf("data size = %lf [MiB]\n", (double)(data_size)/pow(2,20));
sprintf(cname, "simple_compressed_%d.zst", data_size);
sprintf(sname, "simple_raw_%d.dat", data_size);
srand((unsigned int)(time(NULL)));
byte_t* a = (byte_t*)(malloc(data_size));
rand_byte_t(data_size / sizeof(byte_t), a);
compress(data_size, a);
raw_write(data_size, a);
byte_t* b = (byte_t*)(malloc(data_size));
byte_t* c = (byte_t*)(malloc(data_size));
decompress(data_size, b);
raw_read(data_size, c);
for (uint64_t i = 0; i < data_size; ++i)
{
if (a[i] != b[i])
{
fprintf(stderr, "mismatch!\n");
exit(1);
}
}
free(a);
free(b);
free(c);
return 0;
}
void compress(uint64_t data_size, const byte_t* a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open(cbsl_store_mode)\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_write(ctx, a, data_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_write(uint64_t data_size, const byte_t* a)
{
FILE* fp = fopen(sname, "wb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen(wb)\n");
exit(1);
}
fwrite(a, 1, data_size, fp);
fclose(fp);
}
void decompress(uint64_t data_size, byte_t* a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open(cbsl_load_mode)\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_read(ctx, a, data_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_read(uint64_t data_size, byte_t* a)
{
FILE* fp = fopen(sname, "rb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen(rb)\n");
exit(1);
}
fread(a, 1, data_size, fp);
fclose(fp);
}
void rand_byte_t(uint64_t data_size, byte_t* a)
{
for(uint64_t i = 0; i < data_size; ++i)
a[i] = rand() % 255;
}

93
cachelab/cbsl/tests/split_compression.c Normal file
View File

@@ -0,0 +1,93 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
#define CHECK_BINARY(X,Y) {if (memcmp(&(X),&(Y),sizeof((X))) != 0) { fprintf(stderr, "error: binary check %s == %s\n", (#X), (#Y)); exit(1); }}
typedef unsigned char byte_t;
extern void record(cbsl_mode, int, byte_t*, int);
extern void rand_byte_t(int n, byte_t* a);
char cname[128];
int main(int argc, char** argv)
{
if (argc < 3)
return 1;
sprintf(cname, "split.zst");
int block_size, total_size;
sscanf(argv[1], "%d\n", &block_size);
sscanf(argv[2], "%d\n", &total_size);
if (total_size < block_size)
return 2;
byte_t* a = malloc(sizeof(byte_t) * total_size);
byte_t* b = malloc(sizeof(byte_t) * total_size);
rand_byte_t(total_size, a);
record(cbsl_store_mode, total_size, a, block_size);
record(cbsl_load_mode, total_size, b, block_size);
for (int i = 0; i < total_size; ++i)
{
if (a[i] != b[i])
{
fprintf(stderr, "mismatch!\n");
exit(1);
}
}
return 0;
}
void record(cbsl_mode mode, int total_size, byte_t* a, int block_size)
{
cbsl_ctx* ctx = cbsl_open(mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
int total = 0;
do
{
int size = (total_size - total < block_size) ? total_size - total : block_size;
CBSL_ERROR_CHECK(cbsl_record(ctx, a + total, size));
total += size;
}
while(total < total_size);
if (total != total_size)
{
fprintf(stderr, "total != total_size\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void rand_byte_t(int n, byte_t* a) {
for(int i = 0; i < n; ++i)
a[i] = rand() % 255;
}

43
cachelab/cbsl/tests/utility.c Normal file
View File

@@ -0,0 +1,43 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
#define CHECK(X) {if (!(X)) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
char cname[128];
int main(int argc, char** argv)
{
sprintf(cname, "data.zst");
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open\n");
exit(1);
}
CHECK(cbsl_get_mode(ctx) == cbsl_store_mode);
CBSL_ERROR_CHECK(cbsl_set_compression_level(ctx, 20));
CHECK(cbsl_get_compression_level(ctx) == 20);
CBSL_ERROR_CHECK(cbsl_close(ctx));
return 0;
}

155
cachelab/cbsl/tests/variable_size_array.c Normal file
View File

@@ -0,0 +1,155 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <math.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
typedef unsigned char byte_t;
extern void compress(uint64_t data_size, const byte_t* a);
extern void raw_write(uint64_t data_size, const byte_t* a);
extern void decompress(uint64_t* data_size, byte_t** a);
extern void raw_read(uint64_t* data_size, byte_t** a);
extern uint64_t rand_size();
extern void rand_byte_t(uint64_t data_size, byte_t* a);
char cname[128], sname[128];
int main(int argc, char** argv)
{
sprintf(cname, "varsize_compressed.zst");
sprintf(sname, "varsize_raw.dat");
srand((unsigned int)(time(NULL)));
uint64_t data_size;
if (argc >= 2)
{
sscanf(argv[1], "%lu\n", &data_size);
printf("specified data size: %lu byte\n", data_size);
}
else
{
data_size = rand_size();
printf("random generate data size: %lu byte\n", data_size);
}
byte_t* a = (byte_t*)(malloc(data_size));
rand_byte_t(data_size / sizeof(byte_t), a);
compress(data_size, a);
raw_write(data_size, a);
uint64_t b_data_size; byte_t* b;
uint64_t c_data_size; byte_t* c;
decompress(&b_data_size, &b);
raw_read(&c_data_size, &c);
if (data_size != b_data_size || b_data_size != c_data_size)
{
fprintf(stderr, "data size is mismatch!\n");
exit(1);
}
for (uint64_t i = 0; i < data_size; ++i)
{
if (a[i] != b[i] || b[i] != c[i])
{
fprintf(stderr, "data value is mismatch!\n");
exit(1);
}
}
free(a);
free(b);
free(c);
return 0;
}
void compress(uint64_t data_size, const byte_t* a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open(cbsl_store_mode)\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_write(ctx, &data_size, sizeof(data_size)));
CBSL_ERROR_CHECK(cbsl_write(ctx, a, data_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_write(uint64_t data_size, const byte_t* a)
{
FILE* fp = fopen(sname, "wb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen(wb)\n");
exit(1);
}
fwrite(&data_size, 1, sizeof(data_size), fp);
fwrite(a, 1, data_size, fp);
fclose(fp);
}
void decompress(uint64_t* data_size, byte_t** a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open(cbsl_load_mode)\n");
exit(1);
}
CBSL_ERROR_CHECK(cbsl_read(ctx, data_size, sizeof(data_size)));
*a = (byte_t*)(malloc(sizeof(byte_t) * *data_size));
CBSL_ERROR_CHECK(cbsl_read(ctx, *a, *data_size));
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_read(uint64_t* data_size, byte_t** a)
{
FILE* fp = fopen(sname, "rb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen(rb)\n");
exit(1);
}
fread(data_size, 1, sizeof(data_size), fp);
*a = (byte_t*)(malloc(sizeof(byte_t) * *data_size));
fread(*a, 1, *data_size, fp);
fclose(fp);
}
uint64_t rand_size()
{
const uint64_t min_data_size = 1024;
const uint64_t max_data_size = 1048576; /* 1 MiB */
uint64_t size = rand() % max_data_size;
return (size < min_data_size) ? min_data_size : size;
}
void rand_byte_t(uint64_t data_size, byte_t* a)
{
for(uint64_t i = 0; i < data_size; ++i)
a[i] = rand() % 255;
}

189
cachelab/cbsl/tests/various_size_array.c Normal file
View File

@@ -0,0 +1,189 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <math.h>
#include <cbsl.h>
#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); exit(1); }}
typedef unsigned char byte_t;
extern void compress(uint64_t n, uint64_t* sizes, byte_t** a);
extern void raw_write(uint64_t n, uint64_t* sizes, byte_t** a);
extern void decompress(uint64_t n, uint64_t* sizes, byte_t** a);
extern void raw_read(uint64_t n, uint64_t* sizes, byte_t** a);
extern uint64_t rand_size();
extern void rand_byte_t(uint64_t data_size, byte_t* a);
char cname[128], sname[128];
int main(int argc, char** argv)
{
sprintf(cname, "various_compressed.zst");
sprintf(sname, "various_raw.dat");
srand((unsigned int)(time(NULL)));
uint64_t num_vars;
if (argc < 2)
{
num_vars = rand() % 100 + 2;
}
else
{
sscanf(argv[1], "%lu\n", &num_vars);
}
printf("number of array (data sets): %lu\n", num_vars);
byte_t** a = (byte_t**)(malloc(sizeof(byte_t**) * num_vars));
byte_t** b = (byte_t**)(malloc(sizeof(byte_t**) * num_vars));
byte_t** c = (byte_t**)(malloc(sizeof(byte_t**) * num_vars));
uint64_t* asizes = (uint64_t*)(malloc(sizeof(uint64_t*) * num_vars));
uint64_t* bsizes = (uint64_t*)(malloc(sizeof(uint64_t*) * num_vars));
uint64_t* csizes = (uint64_t*)(malloc(sizeof(uint64_t*) * num_vars));
for(uint64_t i = 0; i < num_vars; ++i)
{
asizes[i] = rand_size();
a[i] = (byte_t*)(malloc(sizeof(byte_t*) * asizes[i]));
rand_byte_t(asizes[i] / sizeof(byte_t), a[i]);
printf("array[%lu] = %lu bytes\n", i, asizes[i]);
b[i] = (byte_t*)(malloc(sizeof(byte_t*) * asizes[i]));
c[i] = (byte_t*)(malloc(sizeof(byte_t*) * asizes[i]));
}
compress(num_vars, asizes, a);
raw_write(num_vars, asizes, a);
decompress(num_vars, bsizes, b);
raw_read(num_vars, csizes, c);
for (uint64_t i = 0; i < num_vars; ++i)
{
if (asizes[i] != bsizes[i] || bsizes[i] != csizes[i])
{
fprintf(stderr, "array size is mismatch!\n");
exit(1);
}
}
for (uint64_t i = 0; i < num_vars; ++i)
for (uint64_t j = 0; j < asizes[i]; ++j)
{
if (a[i][j] != b[i][j] || b[i][j] != c[i][j])
{
fprintf(stderr, "data value is mismatch! (a,b,c)[%lu][%lu]\n", i, j);
exit(1);
}
}
for (uint64_t i = 0; i < num_vars; ++i)
{
free(a[i]);
free(b[i]);
free(c[i]);
}
free(a);
free(b);
free(c);
free(asizes);
free(bsizes);
free(csizes);
return 0;
}
void compress(uint64_t n, uint64_t* sizes, byte_t** a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open(cbsl_store_mode)\n");
exit(1);
}
for (uint64_t i = 0; i < n; ++i)
{
CBSL_ERROR_CHECK(cbsl_write(ctx, &sizes[i], sizeof(sizes[i])));
CBSL_ERROR_CHECK(cbsl_write(ctx, a[i], sizes[i]));
}
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_write(uint64_t n, uint64_t* sizes, byte_t** a)
{
FILE* fp = fopen(sname, "wb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen(wb)\n");
exit(1);
}
for (uint64_t i = 0; i < n; ++i)
{
fwrite(&sizes[i], 1, sizeof(sizes[i]), fp);
fwrite(a[i], 1, sizes[i], fp);
}
fclose(fp);
}
void decompress(uint64_t n, uint64_t* sizes, byte_t** a)
{
cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, cname);
if (ctx == NULL)
{
fprintf(stderr, "error: cbsl_open(cbsl_load_mode)\n");
exit(1);
}
for (uint64_t i = 0; i < n; ++i)
{
CBSL_ERROR_CHECK(cbsl_read(ctx, &sizes[i], sizeof(sizes[i])));
CBSL_ERROR_CHECK(cbsl_read(ctx, a[i], sizes[i]));
}
CBSL_ERROR_CHECK(cbsl_close(ctx));
}
void raw_read(uint64_t n, uint64_t* sizes, byte_t** a)
{
FILE* fp = fopen(sname, "rb");
if (fp == NULL)
{
fprintf(stderr, "error: fopen(rb)\n");
exit(1);
}
for (uint64_t i = 0; i < n; ++i)
{
fread(&sizes[i], 1, sizeof(sizes[i]), fp);
fread(a[i], 1, sizes[i], fp);
}
fclose(fp);
}
uint64_t rand_size()
{
const uint64_t min_data_size = 1024;
const uint64_t max_data_size = 1048576; /* 1 MiB */
uint64_t size = rand() % max_data_size;
return (size < min_data_size) ? min_data_size : size;
}
void rand_byte_t(uint64_t data_size, byte_t* a)
{
for(uint64_t i = 0; i < data_size; ++i)
a[i] = rand() % 255;
}