gh0s7/csapp2025
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cachelab started

Browse Source
This commit is contained in:
CGH0S7
2025-04-21 23:52:27 +08:00
parent cc99d9b5d9
commit ace7a46fb9
58 changed files with 10071 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
cachelab/cbsl/src/CMakeLists.txt Normal file
View File

@@ -0,0 +1,46 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set(LIB_SOURCES
buffer.c
file.c
utils.c
flush.c
write.c
read.c
record.c
)
set(FLIB_SOURCES
${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
)
configure_file(fortran_bindings.f90.in ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90.in.2)
find_program(CPP cpp)
if (CPP_NOTFOUND)
message(FATAL_ERROR "C preprocessor (cpp) command not found.")
endif ()
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
COMMAND ${CPP} ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90.in.2 ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_fortran_bindings.cmake ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90.in.2
)
add_custom_target(generate_preprocessed_fortran_file SOURCES ${CMAKE_CURRENT_BINARY_DIR}/fortran_bindings.f90)
add_library(${CBSL_LIB} ${LIB_SOURCES})
add_library(${CBSL_FLIB} ${FLIB_SOURCES})
install(TARGETS ${CBSL_LIB} ${CBSL_FLIB})
install(FILES "${CMAKE_Fortran_MODULE_DIRECTORY}/${CBSL_FLIB}.mod" DESTINATION include)

85
cachelab/cbsl/src/buffer.c Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "./cbsl_internal.h"
cbsl_errors create_streaming_buffers(cbsl_ctx* ctx)
{
ZSTD_CCtx* zstd_cctx = NULL;
ZSTD_DCtx* zstd_dctx = NULL;
uint64_t in_buffer_size;
uint64_t out_buffer_size;
void* in_buffer = NULL;
void* out_buffer = NULL;
void* line_buffer = NULL;
switch (ctx->mode)
{
case cbsl_load_mode:
zstd_dctx = ZSTD_createDCtx();
if (zstd_dctx == NULL) return cbsl_error;
in_buffer_size = ZSTD_DStreamInSize();
out_buffer_size = ZSTD_DStreamOutSize();
break;
case cbsl_store_mode:
zstd_cctx = ZSTD_createCCtx();
if (zstd_cctx == NULL) return cbsl_error;
in_buffer_size = ZSTD_CStreamInSize();
out_buffer_size = ZSTD_CStreamOutSize();
break;
default:
return cbsl_error;
}
in_buffer = (byte_t*)(malloc(in_buffer_size));
out_buffer = (byte_t*)(malloc(out_buffer_size));
line_buffer = (byte_t*)(malloc(CBSL_LINEBUF_SIZE));
if (in_buffer == NULL || out_buffer == NULL || line_buffer == NULL)
{
CBSL_SAFE_FREE_ZSTD_CCTX(zstd_cctx);
CBSL_SAFE_FREE_ZSTD_DCTX(zstd_dctx);
CBSL_SAFE_FREE(in_buffer);
CBSL_SAFE_FREE(out_buffer);
CBSL_SAFE_FREE(line_buffer);
return cbsl_error;
}
ctx->zstd_cctx = zstd_cctx;
ctx->zstd_dctx = zstd_dctx;
ctx->in_buffer = in_buffer;
ctx->out_buffer = out_buffer;
ctx->line_buffer = line_buffer;
ctx->in_buffer_size = in_buffer_size;
ctx->out_buffer_size = out_buffer_size;
ctx->in_buffer_pos = 0;
ctx->out_buffer_pos = 0;
ctx->in_buffer_used = 0;
ctx->out_buffer_used = 0;
ctx->line_buffer_used = 0;
ctx->line_buffer_read_from_zst = 0;
ctx->zstd_buf_end = 0;
ctx->zstd_file_end = 0;
return cbsl_success;
}

78
cachelab/cbsl/src/cbsl_internal.h Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CBSL_INTERNAL_HEADER_INCLUDED
#define CBSL_INTERNAL_HEADER_INCLUDED
#include <stdio.h>
#include <stdint.h>
#include <zstd.h>
#include <cbsl.h>
#define CBSL_LINEBUF_SIZE 8192
typedef unsigned char byte_t;
struct cbsl_ctx_
{
FILE* fp;
ZSTD_CCtx* zstd_cctx;
ZSTD_DCtx* zstd_dctx;
byte_t* in_buffer;
byte_t* out_buffer;
uint64_t in_buffer_size;
uint64_t out_buffer_size;
uint64_t in_buffer_pos;
uint64_t in_buffer_used;
uint64_t out_buffer_pos;
uint64_t out_buffer_used;
cbsl_mode mode;
// support readline
byte_t zstd_buf_end;
byte_t zstd_file_end;
byte_t* line_buffer;
uint64_t line_buffer_used;
uint64_t line_buffer_read_from_zst;
};
cbsl_errors create_streaming_buffers(cbsl_ctx* ctx);
#define MIN(x, y) (x) < (y) ? (x) : (y)
#ifdef CBSL_DEBUG
# include <assert.h>
# define CBSL_ASSERT(COND) { assert((COND)); }
# define CBSL_DEBUG_MESSAGE(...) { fprintf(stderr, __VA_ARGS__); }
#else
# define CBSL_ASSERT(COND) /* */
# define CBSL_DEBUG_MESSAGE(...) /* */
#endif
/* error check */
#define CBSL_CHECK_COND_MSG_IMPL(X,MSG,FINALIZE) { if (!(X)) { fprintf(stderr, "%s at %s l.%d: condition error %s; %s\n", __func__, __FILE__, __LINE__, (#X), (MSG)); FINALIZE; } }
#define CBSL_CHECK_COND(X) CBSL_CHECK_COND_MSG_IMPL((X), "", /* noreturn */)
#define CBSL_CHECK_COND_AND_RETURN(X,RETCODE) CBSL_CHECK_COND_MSG_IMPL((X), "", { return (RETCODE); })
#define CBSL_CHECK_COND_MSG_AND_RETURN(X,MSG,RETCODE) CBSL_CHECK_COND_MSG_IMPL((X), (MSG), { return (RETCODE); })
/* safe memory control */
#define CBSL_SAFE_FREE_RESOURCE(X,FREEFUNC) { if ((X) != NULL) { (FREEFUNC)((X)); (X) = NULL; } }
#define CBSL_SAFE_FCLOSE(X) CBSL_SAFE_FREE_RESOURCE((X), fclose);
#define CBSL_SAFE_FREE(X) CBSL_SAFE_FREE_RESOURCE((X), free)
#define CBSL_SAFE_FREE_ZSTD_CCTX(X) CBSL_SAFE_FREE_RESOURCE((X), ZSTD_freeCCtx)
#define CBSL_SAFE_FREE_ZSTD_DCTX(X) CBSL_SAFE_FREE_RESOURCE((X), ZSTD_freeDCtx)
#endif /* CBSL_INTERNAL_HEADER_INCLUDED */

94
cachelab/cbsl/src/file.c Normal file
View File

@@ -0,0 +1,94 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "./cbsl_internal.h"
static
cbsl_ctx* cbsl_open_safe_finalize(cbsl_ctx** ctx)
{
if (*ctx != NULL)
{
cbsl_close(*ctx);
}
return NULL;
}
cbsl_ctx* cbsl_open(cbsl_mode mode, char* path)
{
CBSL_CHECK_COND_AND_RETURN(path != NULL, NULL);
cbsl_ctx* ctx = (cbsl_ctx*)(malloc(sizeof(cbsl_ctx)));
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, NULL);
char const* fopen_mode;
switch (mode)
{
case cbsl_load_mode: fopen_mode = "rb"; break;
case cbsl_store_mode: fopen_mode = "wb"; break;
default: free(ctx); return NULL;
}
#ifdef _WIN32
errno_t err;
FILE* fp;
err = fopen_s(&fp, path, fopen_mode);
CBSL_CHECK_COND_AND_RETURN(err == 0, cbsl_open_safe_finalize(&ctx));
#else
FILE* fp = fopen(path, fopen_mode);
CBSL_CHECK_COND_AND_RETURN(fp != NULL, cbsl_open_safe_finalize(&ctx));
#endif
ctx->mode = mode;
ctx->fp = fp;
CBSL_CHECK_COND_AND_RETURN(create_streaming_buffers(ctx) == cbsl_success, cbsl_open_safe_finalize(&ctx));
/*
uint64_t file_version;
if (mode == cbsl_load_mode)
{
file_version = 0;
CBSL_CHECK_COND_AND_RETURN(cbsl_read(ctx, &file_version, sizeof(file_version)) == cbsl_success, cbsl_open_safe_finalize(&ctx));
CBSL_CHECK_COND_AND_RETURN(file_version == CBSL_VERSION, cbsl_open_safe_finalize(&ctx));
}
else if (mode == cbsl_store_mode)
{
file_version = CBSL_VERSION;
CBSL_CHECK_COND_AND_RETURN(cbsl_write(ctx, &file_version, sizeof(file_version)) == cbsl_success, cbsl_open_safe_finalize(&ctx));
}
*/
return ctx;
}
cbsl_errors cbsl_close(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_success);
if (ctx->mode == cbsl_store_mode)
{
cbsl_flush(ctx);
}
CBSL_SAFE_FREE_ZSTD_CCTX(ctx->zstd_cctx);
CBSL_SAFE_FREE_ZSTD_DCTX(ctx->zstd_dctx);
CBSL_SAFE_FREE(ctx->in_buffer);
CBSL_SAFE_FREE(ctx->out_buffer);
CBSL_SAFE_FREE(ctx->line_buffer);
CBSL_SAFE_FCLOSE(ctx->fp);
CBSL_SAFE_FREE(ctx);
return cbsl_success;
}

55
cachelab/cbsl/src/flush.c Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "./cbsl_internal.h"
static
cbsl_errors flush_serialize(cbsl_ctx* ctx)
{
if (ctx->in_buffer_used > 0)
{
ZSTD_inBuffer input = { ctx->in_buffer, ctx->in_buffer_used, 0 };
ZSTD_EndDirective mode = (ctx->in_buffer_used < ctx->in_buffer_size) ? ZSTD_e_end : ZSTD_e_continue;
do
{
ZSTD_outBuffer output = { ctx->out_buffer, ctx->out_buffer_size, 0 };
const size_t remaining = ZSTD_compressStream2(ctx->zstd_cctx, &output, &input, mode);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(remaining), ZSTD_getErrorName(remaining), cbsl_error);
const size_t written = fwrite(ctx->out_buffer, 1, output.pos, ctx->fp);
CBSL_CHECK_COND_AND_RETURN(written == output.pos, cbsl_error);
} while (input.pos < input.size);
ctx->in_buffer_used = 0;
}
fflush(ctx->fp);
return cbsl_success;
}
cbsl_errors cbsl_flush(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
if (ctx->mode == cbsl_store_mode)
{
return flush_serialize(ctx);
}
return cbsl_success;
}

471
cachelab/cbsl/src/fortran_bindings.f90.in Normal file
View File

@@ -0,0 +1,471 @@
!
! Copyright 2019 Yuta Hirokawa (University of Tsukuba)
!
! Licensed under the Apache License, Version 2.0 (the "License");
! you may not use this file except in compliance with the License.
! You may obtain a copy of the License at
!
! http://www.apache.org/licenses/LICENSE-2.0
!
! Unless required by applicable law or agreed to in writing, software
! distributed under the License is distributed on an "AS IS" BASIS,
! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
! See the License for the specific language governing permissions and
! limitations under the License.
!
module cbslf
use, intrinsic :: iso_c_binding
implicit none
type cbslf_context
integer(c_intptr_t) :: ctx
end type
public :: cbslf_context
public :: cbslf_store_mode
public :: cbslf_load_mode
public :: cbslf_unknown_mode
public :: cbslf_success
public :: cbslf_error
public :: cbslf_open
public :: cbslf_close
public :: cbslf_flush
public :: cbslf_write
public :: cbslf_read
public :: cbslf_record
public :: cbslf_record_heap
public :: cbslf_get_mode
public :: cbslf_set_compression_level
public :: cbslf_get_compression_level
#define PASTE(a,b) a##b
#define CONCAT(a,b) PASTE(a,b)
#define SET_ERRCODE(CODE) \
if (present(errcode)) errcode = CODE
#define SET_ERRCODE_COND(COND) \
if (present(errcode)) then \
__NL__ if (COND) then \
__NL__ errcode = cbslf_success \
__NL__ else \
__NL__ errcode = cbslf_error \
__NL__ end if \
__NL__ end if
#define FORTRAN_ARRAY_1 :
#define FORTRAN_ARRAY_2 :,:
#define FORTRAN_ARRAY_3 :,:,:
#define FORTRAN_ARRAY_4 :,:,:,:
#define FORTRAN_ARRAY_5 :,:,:,:,:
#define FORTRAN_ARRAY_6 :,:,:,:,:,:
#define FORTRAN_ARRAY_7 :,:,:,:,:,:,:
#define FORTRAN_ARRAY_VALUE_1 1
#define FORTRAN_ARRAY_VALUE_2 1,1
#define FORTRAN_ARRAY_VALUE_3 1,1,1
#define FORTRAN_ARRAY_VALUE_4 1,1,1,1
#define FORTRAN_ARRAY_VALUE_5 1,1,1,1,1
#define FORTRAN_ARRAY_VALUE_6 1,1,1,1,1,1
#define FORTRAN_ARRAY_VALUE_7 1,1,1,1,1,1,1
#define FORTRAN_ARRAY_SHAPE_1 lb(1):ub(1)
#define FORTRAN_ARRAY_SHAPE_2 lb(1):ub(1),lb(2):ub(2)
#define FORTRAN_ARRAY_SHAPE_3 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)
#define FORTRAN_ARRAY_SHAPE_4 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4)
#define FORTRAN_ARRAY_SHAPE_5 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4),lb(5):ub(5)
#define FORTRAN_ARRAY_SHAPE_6 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4),lb(5):ub(5),lb(6):ub(6)
#define FORTRAN_ARRAY_SHAPE_7 lb(1):ub(1),lb(2):ub(2),lb(3):ub(3),lb(4):ub(4),lb(5):ub(5),lb(6):ub(6),lb(7):ub(7)
#define DECLARE_SCALAR_INTERFACE(SAVE_LOAD,ROUTINE_NAME) \
module procedure CONCAT(CONCAT(SAVE_LOAD,_scalar_),ROUTINE_NAME)
#define DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,DIMS) \
module procedure CONCAT(CONCAT(SAVE_LOAD,CONCAT(_array_,DIMS)),CONCAT(_,ROUTINE_NAME))
#define DECLARE_ARRAY_INTERFACE(SAVE_LOAD,ROUTINE_NAME) \
DECLARE_SCALAR_INTERFACE(SAVE_LOAD,ROUTINE_NAME) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,1) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,2) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,3) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,4) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,5) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,6) \
__NL__ DECLARE_ARRAY_INTERFACE_N(SAVE_LOAD,ROUTINE_NAME,7) \
#define DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,DIMS) \
module procedure CONCAT(CONCAT(SAVE_LOAD,CONCAT(_array_heap_,DIMS)),CONCAT(_,ROUTINE_NAME))
#define DECLARE_ARRAY_INTERFACE_ALLOCATABLE(SAVE_LOAD,ROUTINE_NAME) \
DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,1) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,2) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,3) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,4) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,5) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,6) \
__NL__ DECLARE_ARRAY_INTERFACE_ALLOCATABLE_N(SAVE_LOAD,ROUTINE_NAME,7)
#define IMPLEMENT_SCALAR_INTERFACE(ROUTINE_NAME,DATA_TYPE) \
subroutine CONCAT(write_scalar_,ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, target, intent(in) :: val \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(c_int64_t) :: psize \
__NL__ integer(4) :: ret \
__NL__ psize = c_sizeof(val) \
__NL__ ret = cbsl_write(ctx%ctx, loc(val), psize) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(read_scalar_,ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, target, intent(out) :: val \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(c_int64_t) :: psize \
__NL__ integer(4) :: ret \
__NL__ psize = c_sizeof(val) \
__NL__ ret = cbsl_read(ctx%ctx, loc(val), psize) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(record_scalar_,ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, target, intent(inout) :: val \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(c_int64_t) :: psize \
__NL__ integer(4) :: ret \
__NL__ psize = c_sizeof(val) \
__NL__ ret = cbsl_record(ctx%ctx, loc(val), psize) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine
#define IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,DIMS) \
__NL__ subroutine CONCAT(CONCAT(write_array_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, intent(in) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: lb(DIMS),ub(DIMS),nsize,i \
__NL__ integer(c_int64_t) :: bsize \
__NL__ integer(4) :: ret \
__NL__ nsize = size(val, kind=kind(nsize)) \
__NL__ do i=1,size(lb) \
__NL__ lb(i) = lbound(val, i, kind(lb(i))) \
__NL__ ub(i) = ubound(val, i, kind(ub(i))) \
__NL__ end do \
__NL__ call cbslf_write(ctx, nsize, ret) \
__NL__ bsize = size(lb) * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_write(ctx%ctx, loc(lb), bsize), ret) \
__NL__ ret = iand(cbsl_write(ctx%ctx, loc(ub), bsize), ret) \
__NL__ bsize = nsize * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_write(ctx%ctx, loc(val), bsize), ret) \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(CONCAT(read_array_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, intent(out) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: lb(DIMS),ub(DIMS),nsize \
__NL__ integer(c_int64_t) :: bsize \
__NL__ integer(4) :: ret \
__NL__ ret = cbslf_error \
__NL__ if (size(val) > 0) then \
__NL__ call cbslf_read(ctx, nsize, ret) \
__NL__ if (ret == cbslf_success .and. nsize > 0) then \
__NL__ bsize = size(lb) * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(lb), bsize), ret) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(ub), bsize), ret) \
__NL__ bsize = nsize * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(val), bsize), ret) \
__NL__ else \
__NL__ ret = cbslf_error \
__NL__ end if \
__NL__ end if \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(CONCAT(record_array_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, allocatable, intent(inout) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: mode,ret \
__NL__ mode = cbslf_get_mode(ctx) \
__NL__ select case(mode) \
__NL__ case(cbslf_store_mode) \
__NL__ call cbslf_write(ctx, val, ret) \
__NL__ case(cbslf_load_mode) \
__NL__ call cbslf_read(ctx, val, ret) \
__NL__ case default \
__NL__ ret = cbslf_error \
__NL__ end select \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine \
__NL__ subroutine CONCAT(CONCAT(record_array_heap_,CONCAT(DIMS,_)),ROUTINE_NAME)(ctx, val, errcode) \
__NL__ use, intrinsic :: iso_c_binding, only: c_sizeof, c_int64_t \
__NL__ implicit none \
__NL__ type(cbslf_context), intent(in) :: ctx \
__NL__ DATA_TYPE, allocatable, intent(inout) :: val(CONCAT(FORTRAN_ARRAY_,DIMS)) \
__NL__ integer(4), intent(out), optional :: errcode \
__NL__ integer(4) :: lb(DIMS),ub(DIMS),nsize,mode \
__NL__ integer(4) :: ret \
__NL__ integer(c_int64_t) :: bsize \
__NL__ mode = cbslf_get_mode(ctx) \
__NL__ select case(mode) \
__NL__ case(cbslf_store_mode) \
__NL__ call cbslf_write(ctx, val, ret) \
__NL__ case(cbslf_load_mode) \
__NL__ if (allocated(val)) then \
__NL__ call cbslf_read(ctx, val, ret) \
__NL__ else \
__NL__ call cbslf_read(ctx, nsize, ret) \
__NL__ if (ret == cbslf_success .and. nsize > 0) then \
__NL__ bsize = size(lb) * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(lb), bsize), ret) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(ub), bsize), ret) \
__NL__ allocate(val(CONCAT(FORTRAN_ARRAY_SHAPE_,DIMS))) \
__NL__ bsize = nsize * c_sizeof(val(CONCAT(FORTRAN_ARRAY_VALUE_,DIMS))) \
__NL__ ret = iand(cbsl_read(ctx%ctx, loc(val), bsize), ret) \
__NL__ else \
__NL__ ret = cbslf_error \
__NL__ end if \
__NL__ end if \
__NL__ case default \
__NL__ ret = cbslf_error \
__NL__ end select \
__NL__ SET_ERRCODE(ret) \
__NL__ end subroutine
#define IMPLEMENT_ARRAY_INTERFACE(ROUTINE_NAME,DATA_TYPE) \
IMPLEMENT_SCALAR_INTERFACE(ROUTINE_NAME,DATA_TYPE) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,1) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,2) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,3) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,4) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,5) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,6) \
IMPLEMENT_ARRAY_INTERFACE_N(ROUTINE_NAME,DATA_TYPE,7)
!======
private
integer(8), parameter :: CBSL_VERSION = @cbsl_VERSION_MAJOR@@cbsl_VERSION_MINOR@@cbsl_VERSION_PATCH@
character(*), parameter :: CBSL_VERSTION_STRING = '@cbsl_VERSION@'
integer(4), parameter :: CBSL_MAJOR_VERSION = @cbsl_VERSION_MAJOR@
integer(4), parameter :: CBSL_MINOR_VERSION = @cbsl_VERSION_MINOR@
integer(4), parameter :: CBSL_PATCH_VERSION = @cbsl_VERSION_PATCH@
integer(4), parameter :: cbslf_load_mode = 1
integer(4), parameter :: cbslf_store_mode = 2
integer(4), parameter :: cbslf_unknown_mode = -1
integer(4), parameter :: cbslf_success = 0
integer(4), parameter :: cbslf_error = -1
! C function interfaces
interface
function cbsl_open(open_mode, path) result(ctx) &
bind(C,name='cbsl_open')
use, intrinsic :: iso_c_binding
integer(c_int),value, intent(in) :: open_mode
character(kind=c_char),intent(in) :: path(*)
integer(c_intptr_t) :: ctx
end function
function cbsl_close(ctx) result(retcode) &
bind(C,name='cbsl_close')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: retcode
end function
function cbsl_flush(ctx) result(retcode) &
bind(C,name='cbsl_flush')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: retcode
end function
function cbsl_get_mode(ctx) result(retcode) &
bind(C,name='cbsl_get_mode')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: retcode
end function
function cbsl_write(ctx, pdata, psize) result(retcode) &
bind(C,name='cbsl_write')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_intptr_t),value,intent(in) :: pdata
integer(c_int64_t),value,intent(in) :: psize
integer(c_int) :: retcode
end function
function cbsl_read(ctx, pdata, psize) result(retcode) &
bind(C,name='cbsl_read')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_intptr_t),value,intent(in) :: pdata
integer(c_int64_t),value,intent(in) :: psize
integer(c_int) :: retcode
end function
function cbsl_record(ctx, pdata, psize) result(retcode) &
bind(C,name='cbsl_record')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_intptr_t),value,intent(in) :: pdata
integer(c_int64_t),value,intent(in) :: psize
integer(c_int) :: retcode
end function
! cbsl_record_heap() not binds because the function uses malloc()
function cbsl_set_compression_level(ctx,clevel) result(retcode) &
bind(C,name='cbsl_set_compression_level')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int),value,intent(in) :: clevel
integer(c_int) :: retcode
end function
function cbsl_get_compression_level(ctx) result(clevel) &
bind(C,name='cbsl_get_compression_level')
use, intrinsic :: iso_c_binding
integer(c_intptr_t),value,intent(in) :: ctx
integer(c_int) :: clevel
end function
end interface
! Generic routines
interface cbslf_write
DECLARE_SCALAR_INTERFACE(write,character)
DECLARE_SCALAR_INTERFACE(write,logical)
DECLARE_ARRAY_INTERFACE(write,integer4)
DECLARE_ARRAY_INTERFACE(write,integer8)
DECLARE_ARRAY_INTERFACE(write,sreal)
DECLARE_ARRAY_INTERFACE(write,dreal)
DECLARE_ARRAY_INTERFACE(write,ccomplex)
DECLARE_ARRAY_INTERFACE(write,zcomplex)
end interface
interface cbslf_read
DECLARE_SCALAR_INTERFACE(read,character)
DECLARE_SCALAR_INTERFACE(read,logical)
DECLARE_ARRAY_INTERFACE(read,integer4)
DECLARE_ARRAY_INTERFACE(read,integer8)
DECLARE_ARRAY_INTERFACE(read,sreal)
DECLARE_ARRAY_INTERFACE(read,dreal)
DECLARE_ARRAY_INTERFACE(read,ccomplex)
DECLARE_ARRAY_INTERFACE(read,zcomplex)
end interface
interface cbslf_record
DECLARE_SCALAR_INTERFACE(record,character)
DECLARE_SCALAR_INTERFACE(record,logical)
DECLARE_ARRAY_INTERFACE(record,integer4)
DECLARE_ARRAY_INTERFACE(record,integer8)
DECLARE_ARRAY_INTERFACE(record,sreal)
DECLARE_ARRAY_INTERFACE(record,dreal)
DECLARE_ARRAY_INTERFACE(record,ccomplex)
DECLARE_ARRAY_INTERFACE(record,zcomplex)
end interface
interface cbslf_record_heap
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,integer4)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,integer8)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,sreal)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,dreal)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,ccomplex)
DECLARE_ARRAY_INTERFACE_ALLOCATABLE(record,zcomplex)
end interface
contains
IMPLEMENT_SCALAR_INTERFACE(character,character(*,kind=c_char))
IMPLEMENT_SCALAR_INTERFACE(logical,logical(c_bool))
IMPLEMENT_ARRAY_INTERFACE(integer4,integer(c_int32_t))
IMPLEMENT_ARRAY_INTERFACE(integer8,integer(c_int64_t))
IMPLEMENT_ARRAY_INTERFACE(sreal,real(c_float))
IMPLEMENT_ARRAY_INTERFACE(dreal,real(c_double))
IMPLEMENT_ARRAY_INTERFACE(ccomplex,complex(c_float_complex))
IMPLEMENT_ARRAY_INTERFACE(zcomplex,complex(c_double_complex))
function cbslf_open(open_mode, path, errcode) result(ctx)
implicit none
integer(4), intent(in) :: open_mode
character(*), intent(in) :: path
integer(4), intent(out), optional :: errcode
type(cbslf_context) :: ctx
character(1,c_char) :: cpath(len_trim(path)+1)
integer(4) :: n,i
n = len_trim(path, kind=4)
do i=1,n
cpath(i) = path(i:i)
end do
cpath(n+1) = c_null_char
ctx%ctx = cbsl_open(open_mode, cpath)
SET_ERRCODE_COND(ctx%ctx /= 0) ! is not null pointer
end function
subroutine cbslf_close(ctx, errcode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: ret
ret = cbsl_close(ctx%ctx)
SET_ERRCODE(ret)
end subroutine
subroutine cbslf_flush(ctx, errcode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: ret
ret = cbsl_flush(ctx%ctx)
SET_ERRCODE(ret)
end subroutine
function cbslf_get_mode(ctx, errcode) result(open_mode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: open_mode
open_mode = cbsl_get_mode(ctx%ctx)
SET_ERRCODE_COND(open_mode == cbslf_unknown_mode)
end function
subroutine cbslf_set_compression_level(ctx, clevel, errcode)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(in) :: clevel
integer(4), intent(out), optional :: errcode
integer(4) :: ret
ret = cbsl_set_compression_level(ctx%ctx, clevel)
SET_ERRCODE(ret)
end subroutine
function cbslf_get_compression_level(ctx, errcode) result(clevel)
implicit none
type(cbslf_context), intent(in) :: ctx
integer(4), intent(out), optional :: errcode
integer(4) :: clevel
clevel = cbsl_get_compression_level(ctx%ctx)
SET_ERRCODE_COND(clevel > 0)
end function
end module

31
cachelab/cbsl/src/gen_fortran_bindings.cmake Normal file
View File

@@ -0,0 +1,31 @@
#
# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
cmake_minimum_required(VERSION 3.3)
if (${CMAKE_ARGC} LESS 4)
message(FATAL_ERROR "Usage ${CMAKE_ARGV2} <input file> <output file>")
endif ()
set(INPUT_FILE ${CMAKE_ARGV3})
if (CMAKE_ARGV4)
set(OUTPUT_FILE ${CMAKE_ARGV4})
else ()
set(OUTPUT_FILE ${CMAKE_ARGV3})
endif ()
file(READ ${INPUT_FILE} FBIND_SOURCE_TEXT)
string(REGEX REPLACE "__NL__" "\n" FBIND_GEN_TEXT ${FBIND_SOURCE_TEXT})
file(WRITE ${OUTPUT_FILE} ${FBIND_GEN_TEXT})

208
cachelab/cbsl/src/read.c Normal file
View File

@@ -0,0 +1,208 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <zstd.h>
#include "./cbsl_internal.h"
static
cbsl_errors streaming_read_immediate(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
/* previous all decompressed data loaded */
CBSL_CHECK_COND_AND_RETURN(ctx->out_buffer_pos >= ctx->out_buffer_used, cbsl_error);
ZSTD_outBuffer output = { data, size, 0 }; /* decompressed data stores directly */
while (output.pos < output.size)
{
/* previous read data from file */
if (ctx->in_buffer_pos < ctx->in_buffer_used)
{
/* input buffer has remained data from the previous decompression */
memmove(ctx->in_buffer, ctx->in_buffer + ctx->in_buffer_pos, ctx->in_buffer_used - ctx->in_buffer_pos);
ctx->in_buffer_used -= ctx->in_buffer_pos;
}
else
{
ctx->in_buffer_used = 0ULL;
}
ctx->in_buffer_pos = 0ULL;
ctx->in_buffer_used += fread(ctx->in_buffer + ctx->in_buffer_used, 1, ctx->in_buffer_size - ctx->in_buffer_used, ctx->fp);
ZSTD_inBuffer input = { ctx->in_buffer, ctx->in_buffer_used, ctx->in_buffer_pos };
while (input.pos < input.size && output.pos < output.size)
{
const size_t ret = ZSTD_decompressStream(ctx->zstd_dctx, &output, &input);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(ret), ZSTD_getErrorName(ret), cbsl_error);
}
ctx->in_buffer_used = input.size;
ctx->in_buffer_pos = input.pos;
}
CBSL_ASSERT(output.pos == output.size);
ctx->out_buffer_pos = output.pos;
ctx->out_buffer_used = output.size;
return cbsl_success;
}
static
uint64_t read_from_buffer(cbsl_ctx* ctx, byte_t* data, uint64_t size)
{
uint64_t read = 0;
if (ctx->out_buffer_pos < ctx->out_buffer_used)
{
const uint64_t read_size = MIN(ctx->out_buffer_used - ctx->out_buffer_pos, size);
memcpy(data, ctx->out_buffer + ctx->out_buffer_pos, read_size);
ctx->out_buffer_pos += read_size;
read = read_size;
}
return read;
}
static
cbsl_errors streaming_read_buffered(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
size_t zstd_ret = -1;
/* memcpy from the previous decompression data */
uint64_t read = read_from_buffer(ctx, data, size);
byte_t* pdata = (byte_t*)(data);
/* streaming decompression and memcpy */
while (read < size)
{
if (ctx->in_buffer_pos < ctx->in_buffer_used)
{
/* input buffer has remained data from the previous decompression */
memmove(ctx->in_buffer, ctx->in_buffer + ctx->in_buffer_pos, ctx->in_buffer_used - ctx->in_buffer_pos);
ctx->in_buffer_used -= ctx->in_buffer_pos;
}
else
{
/* input buffer is already consumed by the previous decompression */
ctx->in_buffer_used = 0;
}
ctx->in_buffer_pos = 0ULL;
ctx->in_buffer_used += fread(ctx->in_buffer + ctx->in_buffer_used, 1, ctx->in_buffer_size - ctx->in_buffer_used, ctx->fp);
if (ctx->in_buffer_used == 0) // 从文件中，读取不到数据了，无法解压
break;
ZSTD_inBuffer input = { ctx->in_buffer, ctx->in_buffer_used, ctx->in_buffer_pos };
while (input.pos < input.size && read < size)
{
ZSTD_outBuffer output = { ctx->out_buffer, ctx->out_buffer_size, 0 };
zstd_ret = ZSTD_decompressStream(ctx->zstd_dctx, &output, &input);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(zstd_ret), ZSTD_getErrorName(zstd_ret), cbsl_error);
ctx->out_buffer_pos = 0ULL;
ctx->out_buffer_used = output.pos;
read += read_from_buffer(ctx, pdata + read, size - read);
}
ctx->in_buffer_pos = input.pos;
ctx->in_buffer_used = input.size;
if (zstd_ret == 0)
{
// zstd stream end
ctx->zstd_file_end = 1;
break;
}
}
if (ctx->zstd_file_end == 1 && ctx->out_buffer_pos == ctx->out_buffer_used)
ctx->zstd_buf_end = 1;
CBSL_ASSERT(read == size || ctx->zstd_end == 1);
ctx->line_buffer_read_from_zst = read;
return cbsl_success;
}
cbsl_errors cbsl_read(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_load_mode, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_DEBUG_MESSAGE("read: %lu bytes\n", size);
if (size > ctx->out_buffer_size)
{
/* data size is larger than compression buffer */
return streaming_read_immediate(ctx, data, size);
}
else
{
return streaming_read_buffered(ctx, data, size);
}
}
// readline from stream
cbsl_errors cbsl_readline(cbsl_ctx* ctx, char* data, uint64_t size)
{
cbsl_errors ret = cbsl_success;
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_load_mode, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_DEBUG_MESSAGE("%s: %lu bytes\n", __func__, size);
uint64_t line_buffer_space = 0;
line_buffer_space = CBSL_LINEBUF_SIZE - ctx->line_buffer_used;
if (ctx->zstd_buf_end == 0)
{
if (line_buffer_space > ctx->out_buffer_size)
{
/* line_buffer size is larger than compression buffer */
ret = streaming_read_immediate(ctx, ctx->line_buffer + ctx->line_buffer_used, line_buffer_space);
}
else
{
ret = streaming_read_buffered(ctx, ctx->line_buffer + ctx->line_buffer_used, line_buffer_space);
}
ctx->line_buffer_used += ctx->line_buffer_read_from_zst;
}
uint64_t i;
char* ptr;
ptr = (char*)ctx->line_buffer;
data[0] = '\0';
for (i = 0; i < ctx->line_buffer_used; i++)
{
if (*ptr == '\n' || *ptr == '\r')
{
CBSL_CHECK_COND_AND_RETURN(size >= i - 1, cbsl_error);
memcpy(data, ctx->line_buffer, i + 1);
memmove(ctx->line_buffer, ctx->line_buffer + i + 1, ctx->line_buffer_used - i - 1);
ctx->line_buffer_used -= i + 1;
data[i + 1] = '\0';
break;
}
ptr++;
}
if (ctx->line_buffer_used == 0)
return cbsl_end;
else
return ret;
}

70
cachelab/cbsl/src/record.c Normal file
View File

@@ -0,0 +1,70 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdint.h>
#include <stdlib.h>
#include "./cbsl_internal.h"
cbsl_errors cbsl_record(cbsl_ctx* ctx, void* data, uint64_t size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
switch (ctx->mode)
{
case cbsl_load_mode: return cbsl_read(ctx, data, size);
case cbsl_store_mode: return cbsl_write(ctx, data, size);
default: return cbsl_error;
}
}
cbsl_errors cbsl_record_heap(cbsl_ctx* ctx, void** data, uint64_t* size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(size != NULL, cbsl_error);
switch (ctx->mode)
{
case cbsl_load_mode:
{
uint64_t rsize;
void* rdata = *data;
CBSL_CHECK_COND_AND_RETURN(cbsl_read(ctx, &rsize, sizeof(rsize)) == cbsl_success, cbsl_error);
if (rdata == NULL)
{
CBSL_CHECK_COND_AND_RETURN((rdata = malloc(rsize)) != NULL, cbsl_error);
}
else
{
CBSL_CHECK_COND_AND_RETURN(rsize == *size, cbsl_error);
}
CBSL_CHECK_COND_AND_RETURN(cbsl_read(ctx, rdata, rsize) == cbsl_success, cbsl_error);
*size = rsize;
*data = rdata;
}
break;
case cbsl_store_mode:
{
CBSL_CHECK_COND_AND_RETURN(cbsl_write(ctx, size, sizeof(*size)) == cbsl_success, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(cbsl_write(ctx, *data, *size) == cbsl_success, cbsl_error);
}
break;
default:
return cbsl_error;
}
return cbsl_success;
}

49
cachelab/cbsl/src/utils.c Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "./cbsl_internal.h"
/* FIXME: implicit declaration??? */
extern ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
cbsl_mode cbsl_get_mode(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_unknown_mode);
return ctx->mode;
}
cbsl_errors cbsl_set_compression_level(cbsl_ctx* ctx, int clevel)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_store_mode, cbsl_error);
const size_t ret = ZSTD_CCtx_setParameter(ctx->zstd_cctx, ZSTD_c_compressionLevel, clevel);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(ret), ZSTD_getErrorName(ret), cbsl_error);
return cbsl_success;
}
int cbsl_get_compression_level(cbsl_ctx* ctx)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_store_mode, cbsl_error);
int clevel;
const size_t ret = ZSTD_CCtx_getParameter(ctx->zstd_cctx, ZSTD_c_compressionLevel, &clevel);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(ret), ZSTD_getErrorName(ret), cbsl_error);
return clevel;
}

101
cachelab/cbsl/src/write.c Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include "./cbsl_internal.h"
static
cbsl_errors streaming_write_flush(cbsl_ctx* ctx, uint64_t size)
{
if (ctx->in_buffer_used > 0 && ctx->in_buffer_used + size > ctx->in_buffer_size)
{
CBSL_CHECK_COND_AND_RETURN(cbsl_flush(ctx) == cbsl_success, cbsl_error);
}
return cbsl_success;
}
static
cbsl_errors streaming_write_immediate(cbsl_ctx* ctx, const void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
CBSL_CHECK_COND_AND_RETURN(streaming_write_flush(ctx, size) == cbsl_success, cbsl_error);
uint64_t written = 0;
const byte_t* pdata = (const byte_t*)(data);
while (written < size)
{
/* compressing data copies to internal input buffer */
const uint64_t write_size = MIN(size - written, ctx->in_buffer_size);
memcpy(ctx->in_buffer, pdata + written, write_size);
written += write_size;
ZSTD_inBuffer input = { ctx->in_buffer, write_size, 0 };
ZSTD_EndDirective mode = (size - written < ctx->in_buffer_size) ? ZSTD_e_end : ZSTD_e_continue;
do
{
ZSTD_outBuffer output = { ctx->out_buffer, ctx->out_buffer_size, 0 };
const size_t remaining = ZSTD_compressStream2(ctx->zstd_cctx, &output, &input, mode);
CBSL_CHECK_COND_MSG_AND_RETURN(!ZSTD_isError(remaining), ZSTD_getErrorName(remaining), cbsl_error);
CBSL_CHECK_COND_AND_RETURN(fwrite(ctx->out_buffer, 1, output.pos, ctx->fp) == output.pos, cbsl_error);
} while (input.pos < input.size);
CBSL_ASSERT(input.pos == input.size);
}
CBSL_ASSERT(written == size);
return cbsl_success;
}
static
cbsl_errors streaming_write_buffered(cbsl_ctx* ctx, const void* data, uint64_t size)
{
CBSL_DEBUG_MESSAGE("%s: call\n", __func__);
CBSL_CHECK_COND_AND_RETURN(streaming_write_flush(ctx, size) == cbsl_success, cbsl_error);
uint64_t written = 0;
const byte_t* pdata = (const byte_t*)(data);
do
{
const uint64_t write_size = MIN(size - written, ctx->in_buffer_size);
memcpy(ctx->in_buffer, pdata + written, write_size);
ctx->in_buffer_used = write_size;
CBSL_CHECK_COND_AND_RETURN(cbsl_flush(ctx) == cbsl_success, cbsl_error);
written += write_size;
} while (written < size);
CBSL_ASSERT(written == size);
return cbsl_success;
}
cbsl_errors cbsl_write(cbsl_ctx* ctx, const void* data, uint64_t size)
{
CBSL_CHECK_COND_AND_RETURN(ctx != NULL, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(ctx->mode == cbsl_store_mode, cbsl_error);
CBSL_CHECK_COND_AND_RETURN(data != NULL, cbsl_error);
CBSL_DEBUG_MESSAGE("write: %lu bytes\n", size);
if (size > ctx->in_buffer_size)
{
/* data size is larger than compression buffer */
return streaming_write_immediate(ctx, data, size);
}
else
{
return streaming_write_buffered(ctx, data, size);
}
}