build fix

externals/zstd/Makefile

@@ -0,0 +1,443 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+# verbose mode (print commands) on V=1 or VERBOSE=1
+Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
+
+PRGDIR   = programs
+ZSTDDIR  = lib
+BUILDIR  = build
+ZWRAPDIR = zlibWrapper
+TESTDIR  = tests
+FUZZDIR  = $(TESTDIR)/fuzz
+
+# Define nul output
+VOID = /dev/null
+
+# When cross-compiling from linux to windows, you might
+# need to specify this as "Windows." Fedora build fails
+# without it.
+#
+# Note: mingw-w64 build from linux to windows does not
+# fail on other tested distros (ubuntu, debian) even
+# without manually specifying the TARGET_SYSTEM.
+TARGET_SYSTEM ?= $(OS)
+CP ?= cp
+
+ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
+  EXT =.exe
+else
+  EXT =
+endif
+
+## default: Build lib-release and zstd-release
+.PHONY: default
+default: lib-release zstd-release
+
+.PHONY: all
+all: allmost examples manual contrib
+
+.PHONY: allmost
+allmost: allzstd zlibwrapper
+
+# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
+.PHONY: allzstd
+allzstd: lib
+	$(Q)$(MAKE) -C $(PRGDIR) all
+	$(Q)$(MAKE) -C $(TESTDIR) all
+
+.PHONY: all32
+all32:
+	$(MAKE) -C $(PRGDIR) zstd32
+	$(MAKE) -C $(TESTDIR) all32
+
+.PHONY: lib lib-release lib-mt lib-nomt
+lib lib-release lib-mt lib-nomt:
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@
+
+.PHONY: zstd zstd-release
+zstd zstd-release:
+	$(Q)$(MAKE) -C $(PRGDIR) $@
+	$(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT)
+
+.PHONY: zstdmt
+zstdmt:
+	$(Q)$(MAKE) -C $(PRGDIR) $@
+	$(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+
+.PHONY: zlibwrapper
+zlibwrapper: lib
+	$(MAKE) -C $(ZWRAPDIR) all
+
+## test: run long-duration tests
+.PHONY: test
+DEBUGLEVEL ?= 1
+test: MOREFLAGS += -g -Werror
+test:
+	DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
+	$(MAKE) -C $(TESTDIR) $@
+	ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
+
+## shortest: same as `make check`
+.PHONY: shortest
+shortest:
+	$(Q)$(MAKE) -C $(TESTDIR) $@
+
+## check: run basic tests for `zstd` cli
+.PHONY: check
+check: shortest
+
+.PHONY: automated_benchmarking
+automated_benchmarking:
+	$(MAKE) -C $(TESTDIR) $@
+
+.PHONY: benchmarking
+benchmarking: automated_benchmarking
+
+## examples: build all examples in `examples/` directory
+.PHONY: examples
+examples: lib
+	$(MAKE) -C examples all
+
+## manual: generate API documentation in html format
+.PHONY: manual
+manual:
+	$(MAKE) -C contrib/gen_html $@
+
+## man: generate man page
+.PHONY: man
+man:
+	$(MAKE) -C programs $@
+
+## contrib: build all supported projects in `/contrib` directory
+.PHONY: contrib
+contrib: lib
+	$(MAKE) -C contrib/pzstd all
+	$(MAKE) -C contrib/seekable_format/examples all
+	$(MAKE) -C contrib/seekable_format/tests test
+	$(MAKE) -C contrib/largeNbDicts all
+	$(MAKE) -C contrib/externalSequenceProducer all
+	cd build/single_file_libs/ ; ./build_decoder_test.sh
+	cd build/single_file_libs/ ; ./build_library_test.sh
+
+.PHONY: cleanTabs
+cleanTabs:
+	cd contrib; ./cleanTabs
+
+.PHONY: clean
+clean:
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	$(Q)$(MAKE) -C examples/ $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
+	$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
+	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
+	$(Q)$(RM) -r lz4
+	@echo Cleaning completed
+
+#------------------------------------------------------------------------------
+# make install is validated only for Linux, macOS, Hurd and some BSD targets
+#------------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))
+
+HOST_OS = POSIX
+
+MKDIR ?= mkdir -p
+
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($(HAVE_COLORNEVER), 1)
+EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
+
+# Print a two column output of targets and their description. To add a target description, put a
+# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:
+#
+## list: Print all targets and their descriptions (if provided)
+.PHONY: list
+list:
+	$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
+		| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
+		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
+	{ \
+	    printf "Target Name\tDescription\n"; \
+	    printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
+	    for target in $$TARGETS; do \
+	        line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
+	        description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
+	        printf "$$target\t$$description\n"; \
+	    done \
+	} | column -t -s $$'\t'
+
+.PHONY: install armtest usan asan uasan msan asan32
+install:
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@
+	$(Q)$(MAKE) -C $(PRGDIR) $@
+
+.PHONY: uninstall
+uninstall:
+	$(Q)$(MAKE) -C $(ZSTDDIR) $@
+	$(Q)$(MAKE) -C $(PRGDIR) $@
+
+.PHONY: travis-install
+travis-install:
+	$(MAKE) install PREFIX=~/install_test_dir
+
+.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
+gcc5build: clean
+	gcc-5 -v
+	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
+
+gcc6build: clean
+	gcc-6 -v
+	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
+
+gcc7build: clean
+	gcc-7 -v
+	CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
+
+clangbuild: clean
+	clang -v
+	CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
+
+m32build: clean
+	gcc -v
+	$(MAKE) all32
+
+armbuild: clean
+	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd
+
+aarch64build: clean
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd
+
+ppcbuild: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
+
+ppc64build: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
+
+.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
+armfuzz: clean
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+
+aarch64fuzz: clean
+	ld -v
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppcfuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppc64fuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+
+.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
+cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cxxtest: clean
+	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
+
+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)"
+
+armtest: clean
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+
+aarch64test:
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+
+ppctest: clean
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+
+ppc64test: clean
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+
+.PHONY: arm-ppc-compilation
+arm-ppc-compilation:
+	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)"
+
+regressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest
+
+uasanregressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
+
+msanregressiontest:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
+
+update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
+update_regressionResults:
+	$(MAKE) -C programs zstd
+	$(MAKE) -C tests/regression test
+	$(RM) -rf $(REGRESS_RESULTS_DIR)
+	$(MKDIR) $(REGRESS_RESULTS_DIR)
+	./tests/regression/test                         \
+        --cache  tests/regression/cache             \
+        --output $(REGRESS_RESULTS_DIR)/results.csv \
+        --zstd   programs/zstd
+	echo "Showing results differences"
+	! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv
+	echo "Updating results.csv"
+	$(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv
+
+
+# run UBsan with -fsanitize-recover=pointer-overflow
+# this only works with recent compilers such as gcc 8+
+usan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror $(MOREFLAGS)"
+
+asan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
+
+asan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
+
+msan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason
+
+msan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
+
+asan32: clean
+	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
+
+uasan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)"
+
+uasan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
+
+tsan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+
+.PHONY: apt-install
+apt-install:
+	# TODO: uncomment once issue 3011 is resolved and remove hack from Github Actions .yml
+	# sudo apt-get update
+	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
+
+.PHONY: apt-add-repo
+apt-add-repo:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+
+.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
+ppcinstall:
+	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
+
+arminstall:
+	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
+
+valgrindinstall:
+	APT_PACKAGES="valgrind" $(MAKE) apt-install
+
+libc6install:
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
+
+gcc6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
+
+gcc7install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
+
+gcc8install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install
+
+gpp6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
+
+clang38install:
+	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
+
+# Ubuntu 14.04 ships a too-old lz4
+lz4install:
+	[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
+
+endif
+
+
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
+
+ifneq (,$(filter MSYS%,$(shell uname)))
+HOST_OS = MSYS
+CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
+endif
+
+#------------------------------------------------------------------------
+# target specific tests
+#------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
+cmakebuild:
+	cmake --version
+	$(RM) -r $(BUILDIR)/cmake/build
+	$(MKDIR) $(BUILDIR)/cmake/build
+	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
+	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
+	$(MAKE) -C $(BUILDIR)/cmake/build install;
+	$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
+	cd $(BUILDIR)/cmake/build; ctest -V -L Medium
+
+c89build: clean
+	$(CC) -v
+	CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd
+
+gnu90build: clean
+	$(CC) -v
+	CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost
+
+c99build: clean
+	$(CC) -v
+	CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost
+
+gnu99build: clean
+	$(CC) -v
+	CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost
+
+c11build: clean
+	$(CC) -v
+	CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost
+
+bmix64build: clean
+	$(CC) -v
+	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
+
+bmix32build: clean
+	$(CC) -v
+	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
+
+bmi32build: clean
+	$(CC) -v
+	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
+
+# static analyzer test uses clang's scan-build
+# does not analyze zlibWrapper, due to detected issues in zlib source code
+staticAnalyze: SCANBUILD ?= scan-build
+staticAnalyze:
+	$(CC) -v
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd
+endif

externals/zstd/contrib/diagnose_corruption/Makefile

@@ -0,0 +1,35 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+.PHONY: all
+all: check_flipped_bits
+
+ZSTDLIBDIR ?= ../../lib
+
+CFLAGS     ?= -O3
+CFLAGS     += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \
+              -I$(ZSTDLIBDIR)/decompress
+CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+              -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+              -Wstrict-prototypes -Wundef                                     \
+              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+              -Wredundant-decls -Wmissing-prototypes
+CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+
+.PHONY: $(ZSTDLIBDIR)/libzstd.a
+$(ZSTDLIBDIR)/libzstd.a:
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
+
+check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a
+	$(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a
+
+.PHONY: clean
+clean:
+	rm -f check_flipped_bits

externals/zstd/contrib/externalSequenceProducer/Makefile

@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) Yann Collet, Meta Platforms, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common
+
+CFLAGS  ?= -O3
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+default: externalSequenceProducer
+
+all: externalSequenceProducer
+
+externalSequenceProducer: sequence_producer.c main.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) externalSequenceProducer

externals/zstd/contrib/gen_html/Makefile

@@ -0,0 +1,51 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+CXXFLAGS ?= -O3
+CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
+CXXFLAGS += $(MOREFLAGS)
+FLAGS   = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
+
+ZSTDAPI = ../../lib/zstd.h
+ZSTDMANUAL = ../../doc/zstd_manual.html
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LIBVER := $(shell echo $(LIBVER_SCRIPT))
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+
+.PHONY: default
+default: gen_html
+
+.PHONY: all
+all: manual
+
+gen_html: gen_html.cpp
+	$(CXX) $(FLAGS) $^ -o $@$(EXT)
+
+$(ZSTDMANUAL): gen_html $(ZSTDAPI)
+	echo "Update zstd manual in /doc"
+	./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
+
+.PHONY: manual
+manual: gen_html $(ZSTDMANUAL)
+
+.PHONY: clean
+clean:
+	@$(RM) gen_html$(EXT)
+	@echo Cleaning completed

externals/zstd/contrib/largeNbDicts/Makefile

@@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS  ?= -O3
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: largeNbDicts
+
+all : largeNbDicts
+
+largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) largeNbDicts

externals/zstd/contrib/linux-kernel/Makefile

@@ -0,0 +1,108 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+.PHONY: libzstd
+libzstd:
+	rm -rf linux
+	mkdir -p linux
+	mkdir -p linux/include/linux
+	mkdir -p linux/lib/zstd
+	../freestanding_lib/freestanding.py \
+		--source-lib ../../lib \
+		--output-lib linux/lib/zstd \
+		--xxhash '<linux/xxhash.h>' \
+		--xxh64-state 'struct xxh64_state' \
+		--xxh64-prefix 'xxh64' \
+		--rewrite-include '<limits\.h>=<linux/limits.h>' \
+		--rewrite-include '<stddef\.h>=<linux/types.h>' \
+		--rewrite-include '"\.\./zstd.h"=<linux/zstd.h>' \
+		--rewrite-include '"(\.\./)?zstd_errors.h"=<linux/zstd_errors.h>' \
+		--sed 's,/\*\*\*,/* *,g' \
+		--sed 's,/\*\*,/*,g' \
+		--spdx \
+		-DZSTD_NO_INTRINSICS \
+		-DZSTD_NO_UNUSED_FUNCTIONS \
+		-DZSTD_LEGACY_SUPPORT=0 \
+		-DZSTD_STATIC_LINKING_ONLY \
+		-DFSE_STATIC_LINKING_ONLY \
+		-DXXH_STATIC_LINKING_ONLY \
+		-D__GNUC__ \
+		-D__linux__=1 \
+		-DSTATIC_BMI2=0 \
+		-DZSTD_ADDRESS_SANITIZER=0 \
+		-DZSTD_MEMORY_SANITIZER=0 \
+		-DZSTD_DATAFLOW_SANITIZER=0 \
+		-DZSTD_COMPRESS_HEAPMODE=1 \
+		-UNO_PREFETCH \
+		-U__cplusplus \
+		-UZSTD_DLL_EXPORT \
+		-UZSTD_DLL_IMPORT \
+		-U__ICCARM__ \
+		-UZSTD_MULTITHREAD \
+		-U_MSC_VER \
+		-U_WIN32 \
+		-RZSTDLIB_VISIBLE= \
+		-RZSTDERRORLIB_VISIBLE= \
+		-RZSTD_FALLTHROUGH=fallthrough \
+		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
+		-DZSTD_TRACE=0 \
+		-DZSTD_NO_TRACE \
+		-DZSTD_DISABLE_ASM \
+		-DZSTD_LINUX_KERNEL
+	rm linux/lib/zstd/decompress/huf_decompress_amd64.S
+	mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
+	mv linux/lib/zstd/zstd_errors.h linux/include/linux/
+	cp linux_zstd.h linux/include/linux/zstd.h
+	cp zstd_common_module.c linux/lib/zstd
+	cp zstd_compress_module.c linux/lib/zstd
+	cp zstd_decompress_module.c linux/lib/zstd
+	cp decompress_sources.h linux/lib/zstd
+	cp linux.mk linux/lib/zstd/Makefile
+
+LINUX ?= $(HOME)/repos/linux
+
+.PHONY: import
+import: libzstd
+	rm -f $(LINUX)/include/linux/zstd.h
+	rm -f $(LINUX)/include/linux/zstd_errors.h
+	rm -rf $(LINUX)/lib/zstd
+	cp linux/include/linux/zstd.h $(LINUX)/include/linux
+	cp linux/include/linux/zstd_lib.h $(LINUX)/include/linux
+	cp linux/include/linux/zstd_errors.h $(LINUX)/include/linux
+	cp -r linux/lib/zstd $(LINUX)/lib
+
+import-upstream:
+	rm -rf $(LINUX)/lib/zstd
+	mkdir $(LINUX)/lib/zstd
+	cp ../../lib/zstd.h $(LINUX)/include/linux/zstd_lib.h
+	cp -r ../../lib/common $(LINUX)/lib/zstd
+	cp -r ../../lib/compress $(LINUX)/lib/zstd
+	cp -r ../../lib/decompress $(LINUX)/lib/zstd
+	mv $(LINUX)/lib/zstd/zstd_errors.h $(LINUX)/include/linux
+	rm $(LINUX)/lib/zstd/common/threading.*
+	rm $(LINUX)/lib/zstd/common/pool.*
+	rm $(LINUX)/lib/zstd/common/xxhash.*
+	rm $(LINUX)/lib/zstd/compress/zstdmt_*
+
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls -Wmissing-prototypes -Wc++-compat \
+	    -Wimplicit-fallthrough
+
+.PHONY: test
+test: libzstd
+	$(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS) $(DEBUGFLAGS) -Werror" -j
+
+.PHONY: clean
+clean:
+	$(RM) -rf linux
+	$(MAKE) -C test clean

externals/zstd/contrib/linux-kernel/test/Makefile

@@ -0,0 +1,49 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+LINUX := ../linux
+LINUX_ZSTDLIB := $(LINUX)/lib/zstd
+
+CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations
+# Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse
+CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE
+
+LINUX_ZSTD_MODULE     := $(wildcard $(LINUX_ZSTDLIB)/*.c)
+LINUX_ZSTD_COMMON     := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
+LINUX_ZSTD_COMPRESS   := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
+LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c $(LINUX_ZSTDLIB)/decompress/*.S)
+LINUX_ZSTD_FILES      := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS)
+LINUX_ZSTD_OBJECTS0   := $(LINUX_ZSTD_FILES:.c=.o)
+LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_OBJECTS0:.S=.o)
+
+%.o: %.S
+	$(COMPILE.S) $(OUTPUT_OPTION) $<
+
+liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS)
+	$(AR) $(ARFLAGS) $@ $^
+
+test: test.c liblinuxzstd.a
+	$(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
+
+static_test: static_test.c
+	$(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
+
+run-test: test static_test
+	./macro-test.sh
+	./test
+	./static_test
+
+.PHONY:
+clean:
+	$(RM) -f $(LINUX_ZSTDLIB)/*.o
+	$(RM) -f $(LINUX_ZSTDLIB)/**/*.o
+	$(RM) -f *.o *.a
+	$(RM) -f static_test
+	$(RM) -f test

externals/zstd/contrib/pzstd/Makefile

@@ -0,0 +1,268 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+# Standard variables for installation
+DESTDIR ?=
+PREFIX  ?= /usr/local
+BINDIR  := $(DESTDIR)$(PREFIX)/bin
+
+ZSTDDIR = ../../lib
+PROGDIR = ../../programs
+
+# External program to use to run tests, e.g. qemu or valgrind
+TESTPROG  ?=
+# Flags to pass to the tests
+TESTFLAGS ?=
+
+# We use gcc/clang to generate the header dependencies of files
+DEPFLAGS = -MMD -MP -MF $*.Td
+POSTCOMPILE = mv -f $*.Td $*.d
+
+# CFLAGS, CXXFLAGS, CPPFLAGS, and LDFLAGS are for the users to override
+CFLAGS   ?= -O3 -Wall -Wextra
+CXXFLAGS ?= -O3 -Wall -Wextra -pedantic
+CPPFLAGS ?=
+LDFLAGS  ?=
+
+# PZstd uses legacy APIs
+CFLAGS   += -Wno-deprecated-declarations
+
+# Include flags
+PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+GTEST_INC  = -isystem googletest/googletest/include
+
+# If default C++ version is older than C++11, explicitly set C++11, which is the
+# minimum required by the code.
+ifeq ($(shell echo "\043if __cplusplus < 201103L\n\043error\n\043endif" | $(CXX) -x c++ -Werror -c - -o /dev/null 2>/dev/null && echo 1 || echo 0),0)
+PZSTD_CXX_STD := -std=c++11
+endif
+
+PZSTD_CPPFLAGS  = $(PZSTD_INC)
+PZSTD_CCXXFLAGS =
+PZSTD_CFLAGS    = $(PZSTD_CCXXFLAGS)
+PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS) $(PZSTD_CXX_STD)
+PZSTD_LDFLAGS   =
+EXTRA_FLAGS     =
+ALL_CFLAGS      = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS)   $(PZSTD_CFLAGS)
+ALL_CXXFLAGS    = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS)
+ALL_LDFLAGS     = $(EXTRA_FLAGS) $(CXXFLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
+
+
+# gtest libraries need to go before "-lpthread" because they depend on it.
+GTEST_LIB  = -L googletest/build/googlemock/gtest
+LIBS       =
+
+# Compilation commands
+LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
+CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
+CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@
+
+# List all the pzstd source files so we can determine their dependencies
+PZSTD_SRCS  := $(wildcard *.cpp)
+PZSTD_TESTS := $(wildcard test/*.cpp)
+UTILS_TESTS := $(wildcard utils/test/*.cpp)
+ALL_SRCS    := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS)
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+# Standard targets
+.PHONY: default
+default: all
+
+.PHONY: test-pzstd
+test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd: clean googletest pzstd tests check
+
+.PHONY: test-pzstd32
+test-pzstd32: clean googletest32 all32 check
+
+.PHONY: test-pzstd-tsan
+test-pzstd-tsan: LDFLAGS=-fuse-ld=gold
+test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-tsan: clean googletest tsan check
+
+.PHONY: test-pzstd-asan
+test-pzstd-asan: LDFLAGS=-fuse-ld=gold
+test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-asan: clean asan check
+
+.PHONY: check
+check:
+	$(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/ResourcePoolTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS)
+
+.PHONY: install
+install: PZSTD_CPPFLAGS += -DNDEBUG
+install: pzstd$(EXT)
+	install -d -m 755 $(BINDIR)/
+	install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT)
+
+.PHONY: uninstall
+uninstall:
+	$(RM) $(BINDIR)/pzstd$(EXT)
+
+# Targets for many different builds
+.PHONY: all
+all: PZSTD_CPPFLAGS += -DNDEBUG
+all: pzstd$(EXT)
+
+.PHONY: debug
+debug: EXTRA_FLAGS += -g
+debug: pzstd$(EXT) tests roundtrip
+
+.PHONY: tsan
+tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
+tsan: PZSTD_LDFLAGS   += -fsanitize=thread
+tsan: debug
+
+.PHONY: asan
+asan: EXTRA_FLAGS += -fsanitize=address
+asan: debug
+
+.PHONY: ubsan
+ubsan: EXTRA_FLAGS += -fsanitize=undefined
+ubsan: debug
+
+.PHONY: all32
+all32: EXTRA_FLAGS += -m32
+all32: all tests roundtrip
+
+.PHONY: debug32
+debug32: EXTRA_FLAGS += -m32
+debug32: debug
+
+.PHONY: asan32
+asan32: EXTRA_FLAGS += -m32
+asan32: asan
+
+.PHONY: tsan32
+tsan32: EXTRA_FLAGS += -m32
+tsan32: tsan
+
+.PHONY: ubsan32
+ubsan32: EXTRA_FLAGS += -m32
+ubsan32: ubsan
+
+# Run long round trip tests
+.PHONY: roundtripcheck
+roundtripcheck: roundtrip check
+	$(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
+
+# Build the main binary
+pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+	$(LD_COMMAND)
+
+# Target that depends on all the tests
+.PHONY: tests
+tests: EXTRA_FLAGS += -Wno-deprecated-declarations
+tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS)))
+
+# Build the round trip tests
+.PHONY: roundtrip
+roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations
+roundtrip: test/RoundTripTest$(EXT)
+
+# Use the static library that zstd builds for simplicity and
+# so we get the compiler options correct
+.PHONY: $(ZSTDDIR)/libzstd.a
+$(ZSTDDIR)/libzstd.a:
+	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
+
+# Rules to build the tests
+test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o \
+                          $(PROGDIR)/util.o Options.o \
+                          Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+	$(LD_COMMAND)
+
+test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
+test/%Test$(EXT): LIBS += -lgtest -lgtest_main
+test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o \
+                  $(PROGDIR)/util.o Options.o Pzstd.o \
+                  SkippableFrame.o $(ZSTDDIR)/libzstd.a
+	$(LD_COMMAND)
+
+utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
+utils/test/%Test$(EXT): LIBS += -lgtest -lgtest_main
+utils/test/%Test$(EXT): utils/test/%Test.o
+	$(LD_COMMAND)
+
+
+GTEST_CMAKEFLAGS =
+
+# Install googletest
+.PHONY: googletest
+googletest: PZSTD_CCXXFLAGS += -fPIC
+googletest:
+	@$(RM) -rf googletest
+	@git clone https://github.com/google/googletest
+	@mkdir -p googletest/build
+	@cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE)
+
+.PHONY: googletest32
+googletest32: PZSTD_CCXXFLAGS  += -m32
+googletest32: googletest
+
+.PHONY: googletest-mingw64
+googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles"
+googletest-mingw64: googletest
+
+.PHONY: clean
+clean:
+	$(RM) -f *.o pzstd$(EXT) *.Td *.d
+	$(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d
+	$(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d
+	$(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d
+	$(MAKE) -C $(ZSTDDIR) clean
+	@echo Cleaning completed
+
+
+# Cancel implicit rules
+%.o: %.c
+%.o: %.cpp
+
+# Object file rules
+%.o: %.c
+	$(CC_COMMAND)
+	$(POSTCOMPILE)
+
+$(PROGDIR)/%.o: $(PROGDIR)/%.c
+	$(CC_COMMAND)
+	$(POSTCOMPILE)
+
+%.o: %.cpp
+	$(CXX_COMMAND)
+	$(POSTCOMPILE)
+
+test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
+test/%.o: test/%.cpp
+	$(CXX_COMMAND)
+	$(POSTCOMPILE)
+
+utils/test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
+utils/test/%.o: utils/test/%.cpp
+	$(CXX_COMMAND)
+	$(POSTCOMPILE)
+
+# Dependency file stuff
+.PRECIOUS: %.d test/%.d utils/test/%.d
+
+# Include rules that specify header file dependencies
+-include $(patsubst %,%.d,$(basename $(ALL_SRCS)))

externals/zstd/contrib/recovery/Makefile

@@ -0,0 +1,35 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+.PHONY: all
+all: recover_directory
+
+ZSTDLIBDIR ?= ../../lib
+PROGRAMDIR ?= ../../programs
+
+CFLAGS     ?= -O3
+CFLAGS     += -I$(ZSTDLIBDIR) -I$(PROGRAMDIR)
+CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+              -Wstrict-aliasing=1 -Wswitch-enum                               \
+              -Wstrict-prototypes -Wundef                                     \
+              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+              -Wredundant-decls -Wmissing-prototypes
+CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+
+.PHONY: $(ZSTDLIBDIR)/libzstd.a
+$(ZSTDLIBDIR)/libzstd.a:
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
+
+recover_directory: recover_directory.c $(ZSTDLIBDIR)/libzstd.a $(PROGRAMDIR)/util.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
+.PHONY: clean
+clean:
+	rm -f recover_directory

externals/zstd/contrib/seekable_format/examples/Makefile

@@ -0,0 +1,53 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+# This Makefile presumes libzstd is built, using `make` in / or /lib/
+
+ZSTDLIB_PATH = ../../../lib
+ZSTDLIB_NAME = libzstd.a
+ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
+
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
+
+CFLAGS ?= -O3
+CFLAGS += -g
+
+SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
+
+.PHONY: default all clean test
+
+default: all
+
+all: seekable_compression seekable_decompression seekable_decompression_mem \
+	parallel_processing
+
+$(ZSTDLIB):
+	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
+
+seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+parallel_processing : parallel_processing.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
+
+parallel_compression : parallel_compression.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
+
+clean:
+	@rm -f core *.o tmp* result* *.zst \
+		seekable_compression seekable_decompression \
+		seekable_decompression_mem \
+		parallel_processing parallel_compression
+	@echo Cleaning completed

externals/zstd/contrib/seekable_format/tests/Makefile

@@ -0,0 +1,38 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+# This Makefile presumes libzstd is built, using `make` in / or /lib/
+
+ZSTDLIB_PATH = ../../../lib
+ZSTDLIB_NAME = libzstd.a
+ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
+
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
+
+CFLAGS ?= -O3
+CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \
+          -Wformat=2 -Wstrict-aliasing=1
+
+SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
+
+.PHONY: default clean test
+default: test
+
+test: seekable_tests
+	./seekable_tests
+
+$(ZSTDLIB):
+	$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
+
+seekable_tests : $(SEEKABLE_OBJS)
+
+clean:
+	@$(RM) core *.o tmp* result* *.zst \
+		seekable_tests
+	@echo Cleaning completed

externals/zstd/contrib/seqBench/Makefile

@@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS  ?= -O3 -g
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: seqBench
+
+all : seqBench
+
+seqBench: util.o timefn.o benchfn.o datagen.o xxhash.o seqBench.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) seqBench

externals/zstd/doc/educational_decoder/Makefile

@@ -0,0 +1,62 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+ZSTD ?= zstd   # note: requires zstd installation on local system
+
+UNAME?= $(shell uname)
+ifeq ($(UNAME), SunOS)
+DIFF ?= gdiff
+else
+DIFF ?= diff
+endif
+
+HARNESS_FILES=*.c
+
+MULTITHREAD_LDFLAGS = -pthread
+DEBUGFLAGS= -g -DZSTD_DEBUG=1
+CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+            -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS   ?= -O2
+CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+            -Wstrict-aliasing=1 -Wswitch-enum                               \
+            -Wredundant-decls -Wstrict-prototypes -Wundef                   \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+            -std=c99
+CFLAGS   += $(DEBUGFLAGS)
+CFLAGS   += $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
+
+harness: $(HARNESS_FILES)
+	$(CC) $(FLAGS) $^ -o $@
+
+clean:
+	@$(RM) harness *.o
+	@$(RM) -rf harness.dSYM  # MacOS specific
+
+test: harness
+	#
+	# Testing single-file decompression with educational decoder
+	#
+	@$(ZSTD) -f README.md -o tmp.zst
+	@./harness tmp.zst tmp
+	@$(DIFF) -s tmp README.md
+	@$(RM) tmp*
+	#
+	# Testing dictionary decompression with education decoder
+	#
+	# note : files are presented multiple for training, to reach minimum threshold
+	@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
+                  harness.c zstd_decompress.c zstd_decompress.h README.md \
+                  harness.c zstd_decompress.c zstd_decompress.h README.md \
+                  -o dictionary
+	@$(ZSTD) -f README.md -D dictionary -o tmp.zst
+	@./harness tmp.zst tmp dictionary
+	@$(DIFF) -s tmp README.md
+	@$(RM) tmp* dictionary

externals/zstd/examples/Makefile

@@ -0,0 +1,93 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+LIBDIR =../lib
+CPPFLAGS += -I$(LIBDIR)
+LIB = $(LIBDIR)/libzstd.a
+
+
+.PHONY: default
+default: all
+
+.PHONY: all
+all: simple_compression simple_decompression \
+	multiple_simple_compression\
+	dictionary_compression dictionary_decompression \
+	streaming_compression streaming_decompression \
+	multiple_streaming_compression streaming_memory_usage
+
+$(LIB) :
+	$(MAKE) -C $(LIBDIR) libzstd.a
+
+simple_compression.o: common.h
+simple_compression : $(LIB)
+
+simple_decompression.o: common.h
+simple_decompression : $(LIB)
+
+multiple_simple_compression.o: common.h
+multiple_simple_compression : $(LIB)
+
+dictionary_compression.o: common.h
+dictionary_compression : $(LIB)
+
+dictionary_decompression.o: common.h
+dictionary_decompression : $(LIB)
+
+streaming_compression.o: common.h
+streaming_compression : $(LIB)
+
+multiple_streaming_compression.o: common.h
+multiple_streaming_compression : $(LIB)
+
+streaming_decompression.o: common.h
+streaming_decompression : $(LIB)
+
+streaming_memory_usage.o: common.h
+streaming_memory_usage : $(LIB)
+
+
+.PHONY:clean
+clean:
+	@$(RM) core *.o tmp* result* *.zst \
+        simple_compression simple_decompression \
+        multiple_simple_compression \
+        dictionary_compression dictionary_decompression \
+        streaming_compression streaming_decompression \
+        multiple_streaming_compression streaming_memory_usage
+	@echo Cleaning completed
+
+.PHONY:test
+test: all
+	cp README.md tmp
+	cp Makefile tmp2
+	@echo -- Simple compression tests
+	./simple_compression tmp
+	./simple_decompression tmp.zst
+	./multiple_simple_compression *.c
+	./streaming_decompression tmp.zst > /dev/null
+	@echo -- Streaming memory usage
+	./streaming_memory_usage
+	@echo -- Streaming compression tests
+	./streaming_compression tmp
+	./streaming_decompression tmp.zst > /dev/null
+	@echo -- Edge cases detection
+	! ./streaming_decompression tmp    # invalid input, must fail
+	! ./simple_decompression tmp       # invalid input, must fail
+	touch tmpNull                      # create 0-size file
+	./simple_compression tmpNull
+	./simple_decompression tmpNull.zst # 0-size frame : must work
+	@echo -- Multiple streaming tests
+	./multiple_streaming_compression *.c
+	@echo -- Dictionary compression tests
+	./dictionary_compression tmp2 tmp README.md
+	./dictionary_decompression tmp2.zst tmp.zst README.md
+	$(RM) tmp* *.zst
+	@echo tests completed

externals/zstd/lib/Makefile

@@ -0,0 +1,357 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+# Modules
+ZSTD_LIB_COMPRESSION ?= 1
+ZSTD_LIB_DECOMPRESSION ?= 1
+ZSTD_LIB_DICTBUILDER ?= 1
+ZSTD_LIB_DEPRECATED ?= 0
+
+# Input variables for libzstd.mk
+ifeq ($(ZSTD_LIB_COMPRESSION), 0)
+  ZSTD_LIB_DICTBUILDER = 0
+  ZSTD_LIB_DEPRECATED = 0
+endif
+
+ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
+  ZSTD_LEGACY_SUPPORT = 0
+  ZSTD_LIB_DEPRECATED = 0
+endif
+
+include libzstd.mk
+
+ZSTD_FILES := $(ZSTD_COMMON_FILES) $(ZSTD_LEGACY_FILES)
+
+ifneq ($(ZSTD_LIB_COMPRESSION), 0)
+  ZSTD_FILES += $(ZSTD_COMPRESS_FILES)
+endif
+
+ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
+  ZSTD_FILES += $(ZSTD_DECOMPRESS_FILES)
+endif
+
+ifneq ($(ZSTD_LIB_DEPRECATED), 0)
+  ZSTD_FILES += $(ZSTD_DEPRECATED_FILES)
+endif
+
+ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
+  ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES)
+endif
+
+ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
+ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o)
+ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o)
+
+VERSION := $(ZSTD_VERSION)
+
+# Note: by default, the static library is built single-threaded and dynamic library is built
+# multi-threaded. It is possible to force multi or single threaded builds by appending
+# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
+.PHONY: default
+default: lib-release
+
+CPPFLAGS_DYNLIB  += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
+LDFLAGS_DYNLIB   += -pthread
+CPPFLAGS_STATLIB +=                    # static library build defaults to single-threaded
+
+
+ifeq ($(findstring GCC,$(CCVER)),GCC)
+decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
+endif
+
+
+# macOS linker doesn't support -soname, and use different extension
+# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
+ifeq ($(UNAME), Darwin)
+  SHARED_EXT = dylib
+  SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
+  SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
+  SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+else
+  ifeq ($(UNAME), AIX)
+    SONAME_FLAGS =
+  else
+    SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
+  endif
+  SHARED_EXT = so
+  SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
+  SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
+endif
+
+
+.PHONY: all
+all: lib
+
+
+.PHONY: libzstd.a  # must be run every time
+libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB)
+
+SET_CACHE_DIRECTORY = \
+   +$(MAKE) --no-print-directory $@ \
+    BUILD_DIR=obj/$(HASH_DIR) \
+    CPPFLAGS="$(CPPFLAGS)" \
+    CFLAGS="$(CFLAGS)" \
+    LDFLAGS="$(LDFLAGS)"
+
+ifndef BUILD_DIR
+# determine BUILD_DIR from compilation flags
+
+libzstd.a:
+	$(SET_CACHE_DIRECTORY)
+
+else
+# BUILD_DIR is defined
+
+ZSTD_STATLIB_DIR := $(BUILD_DIR)/static
+ZSTD_STATLIB := $(ZSTD_STATLIB_DIR)/libzstd.a
+ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
+$(ZSTD_STATLIB): ARFLAGS = rcs
+$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR)
+$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ)
+  # Check for multithread flag at target execution time
+	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
+    @echo compiling multi-threaded static library $(LIBVER),\
+    @echo compiling single-threaded static library $(LIBVER))
+	$(AR) $(ARFLAGS) $@ $^
+
+libzstd.a: $(ZSTD_STATLIB)
+	cp -f $< $@
+
+endif
+
+ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
+
+LIBZSTD = dll/libzstd.dll
+$(LIBZSTD): $(ZSTD_FILES)
+	@echo compiling dynamic library $(LIBVER)
+	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@
+
+else  # not Windows
+
+LIBZSTD = libzstd.$(SHARED_EXT_VER)
+.PHONY: $(LIBZSTD)  # must be run every time
+$(LIBZSTD): CPPFLAGS += $(CPPFLAGS_DYNLIB)
+$(LIBZSTD): CFLAGS   += -fPIC -fvisibility=hidden
+$(LIBZSTD): LDFLAGS  += -shared $(LDFLAGS_DYNLIB)
+
+ifndef BUILD_DIR
+# determine BUILD_DIR from compilation flags
+
+$(LIBZSTD):
+	$(SET_CACHE_DIRECTORY)
+
+else
+# BUILD_DIR is defined
+
+ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynamic
+ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD)
+ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
+
+$(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR)
+$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
+# Check for multithread flag at target execution time
+	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
+    @echo compiling multi-threaded dynamic library $(LIBVER),\
+    @echo compiling single-threaded dynamic library $(LIBVER))
+	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	@echo creating versioned links
+	ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
+	ln -sf $@ libzstd.$(SHARED_EXT)
+
+$(LIBZSTD): $(ZSTD_DYNLIB)
+	cp -f $< $@
+
+endif  # ifndef BUILD_DIR
+endif  # if windows
+
+.PHONY: libzstd
+libzstd : $(LIBZSTD)
+
+.PHONY: lib
+lib : libzstd.a libzstd
+
+
+# note : do not define lib-mt or lib-release as .PHONY
+# make does not consider implicit pattern rule for .PHONY target
+
+%-mt : CPPFLAGS_DYNLIB  := -DZSTD_MULTITHREAD
+%-mt : CPPFLAGS_STATLIB := -DZSTD_MULTITHREAD
+%-mt : LDFLAGS_DYNLIB   := -pthread
+%-mt : %
+	@echo multi-threaded build completed
+
+%-nomt : CPPFLAGS_DYNLIB  :=
+%-nomt : LDFLAGS_DYNLIB   :=
+%-nomt : CPPFLAGS_STATLIB :=
+%-nomt : %
+	@echo single-threaded build completed
+
+%-release : DEBUGFLAGS :=
+%-release : %
+	@echo release build completed
+
+
+# Generate .h dependencies automatically
+
+DEPFLAGS = -MT $@ -MMD -MP -MF
+
+$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR)
+	@echo CC $@
+	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
+	@echo CC $@
+	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
+	@echo AS $@
+	$(COMPILE.S) $(OUTPUT_OPTION) $<
+
+$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR)
+	@echo AS $@
+	$(COMPILE.S) $(OUTPUT_OPTION) $<
+
+MKDIR ?= mkdir
+$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
+	$(MKDIR) -p $@
+
+DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d)
+$(DEPFILES):
+
+include $(wildcard $(DEPFILES))
+
+
+# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
+ZSTDMT_FILES = compress/zstdmt_compress.c
+ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
+libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
+libzstd-nomt: LDFLAGS += -shared
+libzstd-nomt: $(ZSTD_NOMT_FILES)
+	@echo compiling single-thread dynamic library $(LIBVER)
+	@echo files : $(ZSTD_NOMT_FILES)
+	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+
+.PHONY: clean
+clean:
+	$(RM) -r *.dSYM   # macOS-specific
+	$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+	$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
+	$(RM) -r obj/*
+	@echo Cleaning library completed
+
+#-----------------------------------------------------------------------------
+# make install is validated only for below listed environments
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
+
+lib: libzstd.pc
+
+HAS_EXPLICIT_EXEC_PREFIX := $(if $(or $(EXEC_PREFIX),$(exec_prefix)),1,)
+
+DESTDIR     ?=
+# directory variables : GNU conventions prefer lowercase
+# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+# support both lower and uppercase (BSD), use uppercase in script
+prefix      ?= /usr/local
+PREFIX      ?= $(prefix)
+exec_prefix ?= $(PREFIX)
+EXEC_PREFIX ?= $(exec_prefix)
+libdir      ?= $(EXEC_PREFIX)/lib
+LIBDIR      ?= $(libdir)
+includedir  ?= $(PREFIX)/include
+INCLUDEDIR  ?= $(includedir)
+
+PCINCDIR := $(patsubst $(PREFIX)%,%,$(INCLUDEDIR))
+PCLIBDIR := $(patsubst $(EXEC_PREFIX)%,%,$(LIBDIR))
+
+# If we successfully stripped off a prefix, we'll add a reference to the
+# relevant pc variable.
+PCINCPREFIX := $(if $(findstring $(INCLUDEDIR),$(PCINCDIR)),,$${prefix})
+PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
+
+# If no explicit EXEC_PREFIX was set by the caller, write it out as a reference
+# to PREFIX, rather than as a resolved value.
+PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})
+
+ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
+  PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+else
+  PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+endif
+
+ifneq (,$(filter $(UNAME),SunOS))
+  INSTALL ?= ginstall
+else
+  INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL)
+INSTALL_DATA    ?= $(INSTALL) -m 644
+
+
+libzstd.pc: libzstd.pc.in
+	@echo creating pkgconfig
+	@sed \
+	        -e 's|@PREFIX@|$(PREFIX)|' \
+	        -e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \
+	        -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
+	        -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
+	        -e 's|@VERSION@|$(VERSION)|' \
+	        -e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \
+	        $< >$@
+
+.PHONY: install
+install: install-pc install-static install-shared install-includes
+	@echo zstd static and shared library installed
+
+.PHONY: install-pc
+install-pc: libzstd.pc
+	[ -e $(DESTDIR)$(PKGCONFIGDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
+	$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
+
+.PHONY: install-static
+install-static:
+	# only generate libzstd.a if it's not already present
+	[ -e libzstd.a ] || $(MAKE) libzstd.a-release
+	[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
+	@echo Installing static library
+	$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
+
+.PHONY: install-shared
+install-shared:
+	# only generate libzstd.so if it's not already present
+	[ -e $(LIBZSTD) ] || $(MAKE) libzstd-release
+	[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
+	@echo Installing shared library
+	$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
+	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+
+.PHONY: install-includes
+install-includes:
+	[ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
+	@echo Installing includes
+	$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) zdict.h $(DESTDIR)$(INCLUDEDIR)
+
+.PHONY: uninstall
+uninstall:
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
+	$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
+	$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
+	$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+	$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
+	@echo zstd libraries successfully uninstalled
+
+endif

externals/zstd/lib/dll/example/Makefile

@@ -0,0 +1,48 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+VOID    := /dev/null
+ZSTDDIR  := ../include
+LIBDIR  := ../static
+DLLDIR  := ../dll
+
+CFLAGS  ?= -O3   # can select custom flags. For example : CFLAGS="-O2 -g" make
+CFLAGS  += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
+           -Wdeclaration-after-statement -Wstrict-prototypes \
+           -Wpointer-arith -Wstrict-aliasing=1
+CFLAGS  += $(MOREFLAGS)
+CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_
+FLAGS   := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+.PHONY: default fullbench-dll fullbench-lib
+
+
+default: all
+
+all: fullbench-dll fullbench-lib
+
+
+fullbench-lib: fullbench.c datagen.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib
+
+fullbench-dll: fullbench.c datagen.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll
+
+clean:
+	@$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \
+	@echo Cleaning completed

externals/zstd/programs/Makefile

@@ -0,0 +1,446 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ##########################################################################
+# zstd : Command Line Utility, supporting gzip-like arguments
+# zstd32 : Same as zstd, but forced to compile in 32-bits mode
+# zstd-nolegacy : zstd without support of decompression of legacy versions
+# zstd-small : minimal zstd without dictionary builder and benchmark
+# zstd-compress : compressor-only version of zstd
+# zstd-decompress : decompressor-only version of zstd
+# ##########################################################################
+
+.PHONY: default
+default: zstd-release
+
+LIBZSTD := ../lib
+
+include $(LIBZSTD)/libzstd.mk
+
+ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
+  ALIGN_LOOP = -falign-loops=32
+else
+  ALIGN_LOOP =
+endif
+
+ZSTDLIB_COMMON_SRC := $(sort $(ZSTD_COMMON_FILES))
+ZSTDLIB_COMPRESS_SRC := $(sort $(ZSTD_COMPRESS_FILES))
+ZSTDLIB_DECOMPRESS_SRC := $(sort $(ZSTD_DECOMPRESS_FILES))
+ZSTDLIB_CORE_SRC := $(sort $(ZSTD_DECOMPRESS_FILES) $(ZSTD_COMMON_FILES) $(ZSTD_COMPRESS_FILES))
+ZDICT_SRC := $(sort $(ZSTD_DICTBUILDER_FILES))
+ZSTDLEGACY_SRC := $(sort $(ZSTD_LEGACY_FILES))
+
+# Sort files in alphabetical order for reproducible builds
+ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
+ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC))
+ZSTDLIB_LOCAL_OBJ0 := $(ZSTDLIB_LOCAL_SRC:.c=.o)
+ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_OBJ0:.S=.o)
+
+ZSTD_CLI_SRC := $(sort $(wildcard *.c))
+ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
+
+ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
+ZSTD_ALL_OBJ0 := $(ZSTD_ALL_SRC:.c=.o)
+ZSTD_ALL_OBJ := $(ZSTD_ALL_OBJ0:.S=.o)
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+  EXT =.exe
+  RES64_FILE = windres/zstd64.res
+  RES32_FILE = windres/zstd32.res
+ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
+    RES_FILE = $(RES64_FILE)
+else
+    RES_FILE = $(RES32_FILE)
+endif
+else
+  EXT =
+endif
+
+# thread detection
+NO_THREAD_MSG := ==> no threads, building without multithreading support
+HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
+HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
+ifeq ($(HAVE_THREAD), 1)
+  THREAD_MSG := ==> building with threading support
+  THREAD_CPP := -DZSTD_MULTITHREAD
+  THREAD_LD := -pthread
+else
+  THREAD_MSG := $(NO_THREAD_MSG)
+endif
+
+# zlib detection
+NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
+HAVE_ZLIB ?= $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
+ifeq ($(HAVE_ZLIB), 1)
+  ZLIB_MSG := ==> building zstd with .gz compression support
+  ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
+  ZLIBLD = -lz
+else
+  ZLIB_MSG := $(NO_ZLIB_MSG)
+endif
+
+# lzma detection
+NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
+HAVE_LZMA ?= $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
+ifeq ($(HAVE_LZMA), 1)
+  LZMA_MSG := ==> building zstd with .xz/.lzma compression support
+  LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
+  LZMALD = -llzma
+else
+  LZMA_MSG := $(NO_LZMA_MSG)
+endif
+
+# lz4 detection
+NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
+HAVE_LZ4 ?= $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
+ifeq ($(HAVE_LZ4), 1)
+  LZ4_MSG := ==> building zstd with .lz4 compression support
+  LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
+  LZ4LD = -llz4
+else
+  LZ4_MSG := $(NO_LZ4_MSG)
+endif
+
+# explicit backtrace enable/disable for Linux & Darwin
+ifeq ($(BACKTRACE), 0)
+  DEBUGFLAGS += -DBACKTRACE_ENABLE=0
+endif
+ifeq (,$(filter Windows%, $(OS)))
+ifeq ($(BACKTRACE), 1)
+  DEBUGFLAGS += -DBACKTRACE_ENABLE=1
+  DEBUGFLAGS_LD += -rdynamic
+endif
+endif
+
+SET_CACHE_DIRECTORY = \
+   +$(MAKE) --no-print-directory $@ \
+    BUILD_DIR=obj/$(HASH_DIR) \
+    CPPFLAGS="$(CPPFLAGS)" \
+    CFLAGS="$(CFLAGS)" \
+    LDFLAGS="$(LDFLAGS)" \
+    LDLIBS="$(LDLIBS)" \
+    ZSTD_ALL_SRC="$(ZSTD_ALL_SRC)"
+
+
+.PHONY: all
+all: zstd zstd-compress zstd-decompress zstd-small
+
+.PHONY: allVariants
+allVariants: all zstd-frugal zstd-nolegacy zstd-dictBuilder
+
+.PHONY: zstd  # must always be run
+zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
+zstd : LDFLAGS += $(THREAD_LD) $(DEBUGFLAGS_LD)
+zstd : LDLIBS += $(ZLIBLD) $(LZMALD) $(LZ4LD)
+zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+ifneq (,$(filter Windows%,$(OS)))
+zstd : $(RES_FILE)
+endif
+
+ifndef BUILD_DIR
+# generate BUILD_DIR from flags
+
+zstd:
+	$(SET_CACHE_DIRECTORY)
+
+else
+# BUILD_DIR is defined
+
+ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ))
+$(BUILD_DIR)/zstd : $(ZSTD_OBJ)
+	@echo "$(THREAD_MSG)"
+	@echo "$(ZLIB_MSG)"
+	@echo "$(LZMA_MSG)"
+	@echo "$(LZ4_MSG)"
+	@echo LINK $@
+	$(CC) $(FLAGS) $^ $(LDLIBS) -o $@$(EXT)
+
+ifeq ($(HAVE_HASH),1)
+SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd$(EXT) 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
+DSTBIN_HASH = $(shell cat zstd$(EXT) 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
+BIN_ISDIFFERENT = $(if $(filter $(SRCBIN_HASH),$(DSTBIN_HASH)),0,1)
+else
+BIN_ISDIFFERENT = 1
+endif
+
+zstd : $(BUILD_DIR)/zstd
+	if [ $(BIN_ISDIFFERENT) -eq 1 ]; then \
+		cp -f $<$(EXT) $@$(EXT); \
+		echo zstd build completed; \
+	else \
+		echo zstd already built; \
+	fi
+
+endif  # BUILD_DIR
+
+
+CLEAN += zstd
+.PHONY: zstd-release
+zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
+zstd-release: DEBUGFLAGS_LD :=
+zstd-release: zstd
+
+CLEAN += zstd32
+zstd32 : CPPFLAGS += $(THREAD_CPP)
+zstd32 : LDFLAGS  += $(THREAD_LD)
+zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+ifneq (,$(filter Windows%,$(OS)))
+zstd32 : $(RES32_FILE)
+endif
+zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
+	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
+## zstd-nolegacy: same scope as zstd, with removed support of legacy formats
+CLEAN += zstd-nolegacy
+zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
+zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
+zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
+	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
+
+.PHONY: zstd-nomt
+zstd-nomt : THREAD_CPP :=
+zstd-nomt : THREAD_LD  :=
+zstd-nomt : THREAD_MSG := - multi-threading disabled
+zstd-nomt : zstd
+
+.PHONY: zstd-nogz
+zstd-nogz : ZLIBCPP :=
+zstd-nogz : ZLIBLD  :=
+zstd-nogz : ZLIB_MSG := - gzip support is disabled
+zstd-nogz : zstd
+
+.PHONY: zstd-noxz
+zstd-noxz : LZMACPP :=
+zstd-noxz : LZMALD  :=
+zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
+zstd-noxz : zstd
+
+## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version)
+.PHONY: zstd-dll
+zstd-dll : LDFLAGS+= -L$(LIBZSTD)
+zstd-dll : LDLIBS += -lzstd
+zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c pool.c threading.c
+zstd-dll : zstd
+
+
+## zstd-pgo: zstd executable optimized with PGO.
+.PHONY: zstd-pgo
+zstd-pgo : LLVM_PROFDATA?=llvm-profdata
+zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
+zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Werror=missing-profile -Wno-error=coverage-mismatch)
+zstd-pgo :
+	$(MAKE) clean HASH_DIR=$(HASH_DIR)
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
+	./zstd -b19i1 $(PROFILE_WITH)
+	./zstd -b16i1 $(PROFILE_WITH)
+	./zstd -b9i2 $(PROFILE_WITH)
+	./zstd -b $(PROFILE_WITH)
+	./zstd -b7i2 $(PROFILE_WITH)
+	./zstd -b5 $(PROFILE_WITH)
+ifndef BUILD_DIR
+	$(RM) zstd obj/$(HASH_DIR)/zstd obj/$(HASH_DIR)/*.o
+else
+	$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
+endif
+	case $(CC) in *clang*) if ! [ -e default.profdata ]; then $(LLVM_PROFDATA) merge -output=default.profdata default*.profraw; fi ;; esac
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_USE_FLAGS)"
+
+## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
+CLEAN += zstd-small zstd-frugal
+zstd-small: CFLAGS = -Os -Wl,-s
+zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+
+CLEAN += zstd-decompress
+zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+
+CLEAN += zstd-compress
+zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+
+## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
+CLEAN += zstd-dictBuilder
+zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
+
+CLEAN += zstdmt
+zstdmt: zstd
+	ln -sf zstd zstdmt
+
+.PHONY: generate_res
+generate_res: $(RES64_FILE) $(RES32_FILE)
+
+ifneq (,$(filter Windows%,$(OS)))
+RC ?= windres
+# https://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+$(RES64_FILE): windres/zstd.rc
+	$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
+$(RES32_FILE): windres/zstd.rc
+	$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-i386
+endif
+
+.PHONY: clean
+clean:
+	$(RM) $(CLEAN) core *.o tmp* result* dictionary *.zst \
+        *.gcda default*.profraw default.profdata have_zlib
+	$(RM) -r obj/*
+	@echo Cleaning completed
+
+MD2ROFF = ronn
+MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
+
+zstd.1: zstd.1.md ../lib/zstd.h
+	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+
+zstdgrep.1: zstdgrep.1.md ../lib/zstd.h
+	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+
+zstdless.1: zstdless.1.md ../lib/zstd.h
+	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+
+.PHONY: man
+man: zstd.1 zstdgrep.1 zstdless.1
+
+.PHONY: clean-man
+clean-man:
+	$(RM) zstd.1
+	$(RM) zstdgrep.1
+	$(RM) zstdless.1
+
+.PHONY: preview-man
+preview-man: clean-man man
+	man ./zstd.1
+	man ./zstdgrep.1
+	man ./zstdless.1
+
+
+# Generate .h dependencies automatically
+
+DEPFLAGS = -MT $@ -MMD -MP -MF
+
+$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR)
+	@echo CC $@
+	$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+$(BUILD_DIR)/%.o : %.S | $(BUILD_DIR)
+	@echo AS $@
+	$(COMPILE.S) $(OUTPUT_OPTION) $<
+
+MKDIR ?= mkdir
+$(BUILD_DIR): ; $(MKDIR) -p $@
+
+DEPFILES := $(ZSTD_OBJ:.o=.d)
+$(DEPFILES):
+
+include $(wildcard $(DEPFILES))
+
+
+
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
+
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($(HAVE_COLORNEVER), 1)
+  EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
+AWK = awk
+
+# Print a two column output of targets and their description. To add a target description, put a
+# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:
+#
+## list: Print all targets and their descriptions (if provided)
+.PHONY: list
+list:
+	TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
+		| $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
+		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
+	{ \
+	    printf "Target Name\tDescription\n"; \
+	    printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
+	    for target in $$TARGETS; do \
+	        line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
+	        description=$$(echo $$line | $(AWK) '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
+	        printf "$$target\t$$description\n"; \
+	    done \
+	} | column -t -s $$'\t'
+
+
+DESTDIR     ?=
+# directory variables : GNU conventions prefer lowercase
+# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+# support both lower and uppercase (BSD), use uppercase in script
+prefix      ?= /usr/local
+PREFIX      ?= $(prefix)
+exec_prefix ?= $(PREFIX)
+bindir      ?= $(exec_prefix)/bin
+BINDIR      ?= $(bindir)
+datarootdir ?= $(PREFIX)/share
+mandir      ?= $(datarootdir)/man
+man1dir     ?= $(mandir)/man1
+
+ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS))
+  MANDIR  ?= $(PREFIX)/man
+  MAN1DIR ?= $(MANDIR)/man1
+else
+  MAN1DIR ?= $(man1dir)
+endif
+
+ifneq (,$(filter $(UNAME),SunOS))
+  INSTALL ?= ginstall
+else
+  INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL)
+INSTALL_SCRIPT  ?= $(INSTALL_PROGRAM)
+INSTALL_DATA    ?= $(INSTALL) -m 644
+INSTALL_MAN     ?= $(INSTALL_DATA)
+
+.PHONY: install
+install:
+	# generate zstd only if not already present
+	[ -e zstd ] || $(MAKE) zstd-release
+	[ -e $(DESTDIR)$(BINDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
+	[ -e $(DESTDIR)$(MAN1DIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(MAN1DIR)/
+	@echo Installing binaries
+	$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
+	ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
+	ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
+	ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
+	$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
+	$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
+	@echo Installing man pages
+	$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
+	ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
+	ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
+	$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
+	$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
+	@echo zstd installation completed
+
+.PHONY: uninstall
+uninstall:
+	$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
+	$(RM) $(DESTDIR)$(BINDIR)/zstdless
+	$(RM) $(DESTDIR)$(BINDIR)/zstdcat
+	$(RM) $(DESTDIR)$(BINDIR)/unzstd
+	$(RM) $(DESTDIR)$(BINDIR)/zstdmt
+	$(RM) $(DESTDIR)$(BINDIR)/zstd
+	$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
+	$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
+	$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
+	$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
+	$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
+	@echo zstd programs successfully uninstalled
+
+endif

externals/zstd/tests/Makefile

@@ -0,0 +1,468 @@
+
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+# datagen : Synthetic and parametrable data generator, for tests
+# fullbench  : Precisely measure speed for each zstd inner functions
+# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
+# fuzzer  : Test tool, to check zstd integrity on target platform
+# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
+# paramgrill : parameter tester for zstd
+# test-zstd-speed.py : script for testing zstd speed difference between commits
+# versionsTest : compatibility test between zstd versions stored on Github (v0.1+)
+# zstreamtest : Fuzzer test tool for zstd streaming API
+# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
+# ##########################################################################
+
+LIBZSTD = ../lib
+
+ZSTD_LEGACY_SUPPORT ?= 0
+
+DEBUGLEVEL ?= 2
+export DEBUGLEVEL  # transmit value to sub-makefiles
+
+include $(LIBZSTD)/libzstd.mk
+
+ZSTDDIR = $(LIBZSTD)
+PRGDIR  = ../programs
+PYTHON ?= python3
+TESTARTEFACT := versionsTest
+
+DEBUGFLAGS += -g -Wno-c++-compat
+CPPFLAGS   += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+              -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
+              -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
+
+ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
+ZSTDCOMP_FILES   := $(sort $(ZSTD_COMPRESS_FILES))
+ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES))
+ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES))
+
+ZSTD_F1 := $(sort $(wildcard $(ZSTD_FILES)))
+ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1))
+ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1))
+ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2))
+ZSTD_OBJ4 := $(ZSTD_OBJ3:.c=.o)
+ZSTD_OBJECTS := $(ZSTD_OBJ4:.S=.o)
+
+ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1))
+ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1))
+ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2))
+ZSTDMT_OBJ4 := $(ZSTDMT_OBJ3:.c=.o)
+ZSTDMT_OBJECTS := $(ZSTDMT_OBJ4:.S=.o)
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  =
+else
+EXT =
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  = -pthread
+endif
+MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
+
+VOID = /dev/null
+ZSTREAM_TESTTIME ?= -T90s
+FUZZERTEST ?= -T200s
+ZSTDRTTEST = --test-large-data
+DECODECORPUS_TESTTIME ?= -T30
+
+.PHONY: default
+default: fullbench
+
+.PHONY: all
+all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash poolTests
+
+.PHONY: all32
+all32: fullbench32 fuzzer32 zstreamtest32
+
+.PHONY: allnothread
+allnothread: MULTITHREAD_CPP=
+allnothread: MULTITHREAD_LD=
+allnothread: fullbench fuzzer paramgrill datagen decodecorpus
+
+# note : broken : requires symbols unavailable from dynamic library
+.PHONY: dll
+dll: fuzzer-dll zstreamtest-dll
+
+.PHONY: zstd zstd32 zstd-nolegacy  # only external makefile knows how to build or update them
+zstd zstd32 zstd-nolegacy zstd-dll:
+	$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
+
+.PHONY: libzstd
+libzstd :
+	$(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS+="$(DEBUGFLAGS)"
+
+%-dll : libzstd
+%-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd
+
+$(ZSTDDIR)/libzstd.a :
+	$(MAKE) -C $(ZSTDDIR) libzstd.a
+
+zstdm_%.o : $(ZSTDDIR)/common/%.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+zstdc_%.o : $(ZSTDDIR)/compress/%.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+zstdd_%.o : $(ZSTDDIR)/decompress/%.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+zstdd_%.o : $(ZSTDDIR)/decompress/%.S
+	$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@
+
+zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP)
+
+zstdmt_m_%.o : $(ZSTDDIR)/common/%.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S
+	$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@
+
+FULLBENCHS := fullbench fullbench32
+CLEAN += $(FULLBENCHS)
+fullbench32: CPPFLAGS += -m32
+$(FULLBENCHS) : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
+$(FULLBENCHS) : LDFLAGS += $(MULTITHREAD_LD)
+$(FULLBENCHS) : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
+$(FULLBENCHS) : $(ZSTD_FILES)
+$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
+	$(LINK.c) $^ -o $@$(EXT)
+
+CLEAN += fullbench-lib
+fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
+fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c
+	$(LINK.c) $^ -o $@$(EXT)
+
+# note : broken : requires symbols unavailable from dynamic library
+fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c
+#	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
+	$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)
+
+CLEAN += fuzzer fuzzer32
+fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
+fuzzer : LDFLAGS += $(MULTITHREAD_LD)
+fuzzer : $(ZSTDMT_OBJECTS)
+fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
+
+fuzzer32 : CFLAGS += -m32 $(MULTITHREAD)
+fuzzer32 : $(ZSTD_FILES)
+	$(LINK.c) $^ -o $@$(EXT)
+
+# note : broken : requires symbols unavailable from dynamic library
+fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
+
+CLEAN += zstreamtest zstreamtest32
+ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c external_matchfinder.c
+ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
+ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
+zstreamtest32 : CFLAGS += -m32
+zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
+zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
+zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES)
+zstreamtest32 : $(ZSTREAMFILES)
+zstreamtest zstreamtest32 :
+	$(LINK.c) $^ -o $@$(EXT)
+
+CLEAN += zstreamtest_asan
+zstreamtest_asan : CFLAGS += -fsanitize=address
+zstreamtest_asan : $(ZSTREAMFILES)
+	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
+CLEAN += zstreamtest_tsan
+zstreamtest_tsan : CFLAGS += -fsanitize=thread
+zstreamtest_tsan : $(ZSTREAMFILES)
+	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
+CLEAN += zstreamtest_ubsan
+zstreamtest_ubsan : CFLAGS += -fsanitize=undefined
+zstreamtest_ubsan : $(ZSTREAMFILES)
+	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
+# note : broken : requires symbols unavailable from dynamic library
+zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c  # xxh symbols not exposed from dll
+zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
+
+CLEAN += paramgrill
+paramgrill : DEBUGFLAGS =   # turn off debug for speed measurements
+paramgrill : LDLIBS += -lm
+paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
+
+CLEAN += datagen
+datagen : $(PRGDIR)/datagen.c datagencli.c
+	$(LINK.c) $^ -o $@$(EXT)
+
+CLEAN += roundTripCrash
+roundTripCrash: CFLAGS += $(MULTITHREAD)
+roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
+
+CLEAN += longmatch
+longmatch : $(ZSTD_OBJECTS) longmatch.c
+
+CLEAN += bigdict
+bigdict: CFLAGS += $(MULTITHREAD)
+bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
+
+CLEAN += invalidDictionaries
+invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
+
+CLEAN += legacy
+legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
+legacy : $(ZSTD_FILES) $(sort $(wildcard $(ZSTDDIR)/legacy/*.c)) legacy.c
+
+CLEAN += decodecorpus
+decodecorpus : LDLIBS += -lm
+decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c
+
+CLEAN += poolTests
+poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
+	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
+.PHONY: versionsTest
+versionsTest: clean
+	$(PYTHON) test-zstd-versions.py
+
+.PHONY: automated_benchmarking
+automated_benchmarking: clean
+	$(PYTHON) automated_benchmarking.py
+
+# make checkTag : check that release tag corresponds to release version
+CLEAN += checkTag
+checkTag.o : $(ZSTDDIR)/zstd.h
+
+.PHONY: clean
+clean:
+	$(MAKE) -C $(ZSTDDIR) clean
+	$(MAKE) -C $(PRGDIR) clean
+	$(RM) -fR $(TESTARTEFACT)
+	$(RM) -rf tmp*  # some test directories are named tmp*
+	$(RM) $(CLEAN) core *.o *.tmp result* *.gcda dictionary *.zst \
+        $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
+        fullbench-dll$(EXT) fuzzer-dll$(EXT) zstreamtest-dll$(EXT)
+	@echo Cleaning completed
+
+
+#----------------------------------------------------------------------------------
+# valgrind tests validated only for some posix platforms
+#----------------------------------------------------------------------------------
+UNAME := $(shell uname)
+ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX))
+HOST_OS = POSIX
+
+.PHONY: test-valgrind
+test-valgrind: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
+test-valgrind: zstd datagen fuzzer fullbench
+	@echo "\n ---- valgrind tests : memory analyzer ----"
+	$(VALGRIND) ./datagen -g50M > $(VOID)
+	$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
+	./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID)
+	./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
+	./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
+	$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
+	./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
+	$(RM) tmp
+	$(VALGRIND) ./fuzzer -T1mn -t1
+	$(VALGRIND) ./fullbench -i1
+
+endif
+
+ifneq (,$(filter MINGW% MSYS%,$(UNAME)))
+  HOST_OS = MSYS
+endif
+
+
+#-----------------------------------------------------------------------------
+# make tests validated only for below targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+
+DIFF:=diff
+ifneq (,$(filter $(UNAME),SunOS))
+  DIFF:=gdiff
+endif
+
+.PHONY: list
+list:
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
+.PHONY: shortest
+shortest: ZSTDRTTEST=  # remove long tests
+shortest: test-zstd
+
+.PHONY: check
+check: shortest
+
+.PHONY: fuzztest
+fuzztest: test-fuzzer test-zstream test-decodecorpus
+
+.PHONY: test
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests
+ifeq ($(QEMU_SYS),)
+test: test-pool
+endif
+
+.PHONY: test32
+test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
+
+.PHONY: test-all
+test-all: test test32 test-decodecorpus-cli
+
+.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
+test-zstd: ZSTD = $(PRGDIR)/zstd
+test-zstd: zstd
+
+.PHONY: test-zstd-dll
+test-zstd-dll: ZSTD = $(PRGDIR)/zstd
+test-zstd-dll: zstd-dll
+
+test-zstd32: ZSTD = $(PRGDIR)/zstd32
+test-zstd32: zstd32
+
+test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
+test-zstd-nolegacy: zstd-nolegacy
+
+test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen
+	file $(ZSTD)
+	EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
+
+.PHONY: test-cli-tests
+test-cli-tests: ZSTD = $(PRGDIR)/zstd
+test-cli-tests: zstd datagen
+	file $(ZSTD)
+	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
+
+.PHONY: test-fullbench
+test-fullbench: fullbench datagen
+	$(QEMU_SYS) ./fullbench -i1
+	$(QEMU_SYS) ./fullbench -i1 -P0
+
+.PHONY: test-fullbench32
+test-fullbench32: fullbench32 datagen
+	$(QEMU_SYS) ./fullbench32 -i1
+	$(QEMU_SYS) ./fullbench32 -i1 -P0
+
+.PHONY: test-fuzzer
+test-fuzzer: fuzzer
+	$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
+
+# Note : this test presumes `fuzzer` will be built
+.PHONY: test-fuzzer-stackmode
+test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0
+test-fuzzer-stackmode: test-fuzzer
+
+.PHONY: test-fuzzer32
+test-fuzzer32: fuzzer32
+	$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
+
+.PHONY: test-zstream
+test-zstream: zstreamtest
+	$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+
+test-zstream32: zstreamtest32
+	$(QEMU_SYS) ./zstreamtest32 -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+
+test-longmatch: longmatch
+	$(QEMU_SYS) ./longmatch
+
+test-bigdict: bigdict
+	$(QEMU_SYS) ./bigdict
+
+test-invalidDictionaries: invalidDictionaries
+	$(QEMU_SYS) ./invalidDictionaries
+
+test-legacy: legacy
+	$(QEMU_SYS) ./legacy
+
+test-decodecorpus: decodecorpus
+	$(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
+
+test-decodecorpus-cli: decodecorpus
+	@echo "\n ---- decodecorpus basic cli tests ----"
+	@mkdir testdir
+	./decodecorpus -n5 -otestdir -ptestdir
+	@cd testdir && \
+	$(ZSTD) -d z000000.zst -o tmp0 && \
+	$(ZSTD) -d z000001.zst -o tmp1 && \
+	$(ZSTD) -d z000002.zst -o tmp2 && \
+	$(ZSTD) -d z000003.zst -o tmp3 && \
+	$(ZSTD) -d z000004.zst -o tmp4 && \
+	diff z000000 tmp0 && \
+	diff z000001 tmp1 && \
+	diff z000002 tmp2 && \
+	diff z000003 tmp3 && \
+	diff z000004 tmp4 && \
+	rm ./* && \
+	cd ..
+	@echo "\n ---- decodecorpus dictionary cli tests ----"
+	./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB
+	@cd testdir && \
+	$(ZSTD) -d z000000.zst -D dictionary -o tmp0 && \
+	$(ZSTD) -d z000001.zst -D dictionary -o tmp1 && \
+	$(ZSTD) -d z000002.zst -D dictionary -o tmp2 && \
+	$(ZSTD) -d z000003.zst -D dictionary -o tmp3 && \
+	$(ZSTD) -d z000004.zst -D dictionary -o tmp4 && \
+	diff z000000 tmp0 && \
+	diff z000001 tmp1 && \
+	diff z000002 tmp2 && \
+	diff z000003 tmp3 && \
+	diff z000004 tmp4 && \
+	cd ..
+	@rm -rf testdir
+
+test-pool: poolTests
+	$(QEMU_SYS) ./poolTests
+
+test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
+test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
+test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
+test-lz4: zstd decodecorpus datagen
+	[ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
+	[ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
+
+	./decodecorpus -ptmp
+	# lz4 -> zstd
+	lz4 < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+	lz4 < tmp | \
+	$(ZSTD_UNLZ4) | \
+	cmp - tmp
+	# zstd -> lz4
+	$(ZSTD) --format=lz4 < tmp | \
+	lz4 -d | \
+	cmp - tmp
+	$(ZSTD_LZ4) < tmp | \
+	lz4 -d | \
+	cmp - tmp
+	# zstd -> zstd
+	$(ZSTD) --format=lz4 < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+	# zstd -> zstd
+	$(ZSTD) < tmp | \
+	$(ZSTD) -d | \
+	cmp - tmp
+
+	./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null
+
+	rm tmp lz4 unlz4
+
+endif

externals/zstd/tests/fuzz/Makefile

@@ -0,0 +1,270 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+# Optionally user defined flags
+CFLAGS ?= -O3
+CXXFLAGS ?= -O3
+CPPFLAGS ?=
+ASFLAGS ?=
+LDFLAGS ?=
+ARFLAGS ?=
+LIB_FUZZING_ENGINE ?= libregression.a
+PYTHON ?= python
+ifeq ($(shell uname), Darwin)
+	DOWNLOAD?=curl -L -o
+else
+	DOWNLOAD?=wget -O
+endif
+CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/
+
+LIBZSTD = ../../lib
+DEBUGLEVEL ?= 2
+ZSTD_LEGACY_SUPPORT ?= 1
+
+include $(LIBZSTD)/libzstd.mk
+
+ZSTDDIR = ../../lib
+PRGDIR = ../../programs
+CONTRIBDIR = ../../contrib
+
+DEFAULT_SEQ_PROD_DIR = $(CONTRIBDIR)/externalSequenceProducer
+DEFAULT_SEQ_PROD_SRC = $(DEFAULT_SEQ_PROD_DIR)/sequence_producer.c
+THIRD_PARTY_SEQ_PROD_OBJ ?=
+
+FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+	-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
+	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(DEFAULT_SEQ_PROD_DIR) \
+	-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
+FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+	-Wstrict-prototypes -Wundef \
+	-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+	-Wredundant-decls -Wno-deprecated-declarations \
+	-g -fno-omit-frame-pointer
+FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS)
+FUZZ_ASFLAGS := $(FUZZ_EXTRA_FLAGS) $(ASFLAGS)
+FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS)
+FUZZ_LDFLAGS := -pthread $(LDFLAGS)
+FUZZ_ARFLAGS := $(ARFLAGS)
+FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
+
+FUZZ_ROUND_TRIP_FLAGS := -DFUZZING_ASSERT_VALID_SEQUENCE
+
+FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h
+FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_producer.c
+
+SEEKABLE_HEADERS = $(CONTRIBDIR)/seekable_format/zstd_seekable.h
+SEEKABLE_OBJS = $(CONTRIBDIR)/seekable_format/zstdseek_compress.c $(CONTRIBDIR)/seekable_format/zstdseek_decompress.c
+
+ZSTDCOMMON_SRC := $(ZSTD_COMMON_FILES)
+ZSTDCOMP_SRC   := $(ZSTD_COMPRESS_FILES)
+ZSTDDECOMP_SRC := $(ZSTD_DECOMPRESS_FILES)
+ZSTDDICT_SRC := $(ZSTD_DICTBUILDER_FILES)
+ZSTDLEGACY_SRC := $(ZSTD_LEGACY_FILES)
+FUZZ_SRC       := \
+	$(FUZZ_SRC) \
+	$(ZSTDDECOMP_SRC) \
+	$(ZSTDCOMMON_SRC) \
+	$(ZSTDCOMP_SRC) \
+	$(ZSTDDICT_SRC) \
+	$(ZSTDLEGACY_SRC) \
+	$(DEFAULT_SEQ_PROD_SRC)
+FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC)))
+
+FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC))
+FUZZ_D_OBJ2 := $(subst $(ZSTDDIR)/compress/,d_lib_compress_,$(FUZZ_D_OBJ1))
+FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2))
+FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3))
+FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
+FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
+FUZZ_D_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,d_default_seq_prod_,$(FUZZ_D_OBJ6))
+FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7))
+FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o)
+FUZZ_D_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_D_OBJ9)
+FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ10:.S=.o)
+
+FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
+FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
+FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2))
+FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3))
+FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
+FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
+FUZZ_RT_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,rt_default_seq_prod_,$(FUZZ_RT_OBJ6))
+FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7))
+FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
+FUZZ_RT_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_RT_OBJ9)
+FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ10:.S=.o)
+
+.PHONY: default all clean cleanall
+
+default: all
+
+FUZZ_TARGETS :=       \
+	simple_round_trip \
+	stream_round_trip \
+	block_round_trip  \
+	simple_decompress \
+	stream_decompress \
+	block_decompress  \
+	dictionary_round_trip \
+	dictionary_decompress \
+	zstd_frame_info \
+	simple_compress \
+	dictionary_loader \
+	raw_dictionary_round_trip \
+	dictionary_stream_round_trip \
+	decompress_dstSize_tooSmall \
+	fse_read_ncount \
+	sequence_compression_api \
+	seekable_roundtrip \
+	huf_round_trip \
+	huf_decompress
+
+all: libregression.a $(FUZZ_TARGETS)
+
+rt_lib_common_%.o: $(ZSTDDIR)/common/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_lib_compress_%.o: $(ZSTDDIR)/compress/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_ASFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_lib_legacy_%.o: $(ZSTDDIR)/legacy/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_prg_%.o: $(PRGDIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_fuzz_%.o: %.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+rt_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
+d_lib_common_%.o: $(ZSTDDIR)/common/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_lib_compress_%.o: $(ZSTDDIR)/compress/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_ASFLAGS) $< -c -o $@
+
+d_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_lib_legacy_%.o: $(ZSTDDIR)/legacy/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_prg_%.o: $(PRGDIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_fuzz_%.o: %.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+d_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
+simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_stream_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+block_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_block_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+simple_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_simple_decompress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@
+
+stream_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_stream_decompress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@
+
+block_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_block_decompress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
+
+dictionary_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+raw_dictionary_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_raw_dictionary_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_raw_dictionary_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+dictionary_stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_stream_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+dictionary_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_dictionary_decompress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_dictionary_decompress.o $(LIB_FUZZING_ENGINE) -o $@
+
+simple_compress: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_compress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_compress.o $(LIB_FUZZING_ENGINE) -o $@
+
+zstd_frame_info: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_zstd_frame_info.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_zstd_frame_info.o $(LIB_FUZZING_ENGINE) -o $@
+
+dictionary_loader: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o $(LIB_FUZZING_ENGINE) -o $@
+
+decompress_dstSize_tooSmall: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_dstSize_tooSmall.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_dstSize_tooSmall.o $(LIB_FUZZING_ENGINE) -o $@
+
+fse_read_ncount: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount.o $(LIB_FUZZING_ENGINE) -o $@
+
+sequence_compression_api: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(LIB_FUZZING_ENGINE) -o $@
+
+seekable_roundtrip: $(FUZZ_HEADERS) $(SEEKABLE_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS)  rt_fuzz_seekable_roundtrip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o $(LIB_FUZZING_ENGINE) -o $@
+
+huf_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_huf_round_trip.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_huf_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
+
+huf_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_huf_decompress.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_huf_decompress.o $(LIB_FUZZING_ENGINE) -o $@
+
+libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o
+	$(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o
+
+corpora/%_seed_corpus.zip:
+	@mkdir -p corpora
+	$(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip
+
+corpora/%: corpora/%_seed_corpus.zip
+	unzip -q $^ -d $@
+
+.PHONY: corpora
+corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS))
+
+.PHONY: seedcorpora
+seedcorpora: $(patsubst %,corpora/%_seed_corpus.zip,$(FUZZ_TARGETS))
+
+regressiontest: corpora
+	CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all
+	$(PYTHON) ./fuzz.py regression all
+
+clean:
+	@$(RM) *.a *.o $(FUZZ_TARGETS)
+	@echo Cleaning completed
+
+cleanall:
+	@$(RM) -r Fuzzer
+	@$(RM) -r corpora
+	@echo Cleaning completed

externals/zstd/tests/fuzz/seq_prod_fuzz_example/Makefile

@@ -0,0 +1,16 @@
+# Copyright (c) Yann Collet, Meta Platforms, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+
+CC = clang
+CFLAGS = -g -fno-omit-frame-pointer -fsanitize=undefined,address,fuzzer -I../ -I../../../lib/
+
+.PHONY: default
+default: example_seq_prod.o
+
+example_seq_prod.o: example_seq_prod.c
+	$(CC) -c $(CFLAGS) $^ -o $@

externals/zstd/tests/gzip/Makefile

@@ -0,0 +1,45 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+PRGDIR = ../../programs
+VOID   = /dev/null
+export PATH := .:$(PATH)
+
+.PHONY: all
+#all: test-gzip-env
+all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed
+all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid
+all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix
+	@echo Testing completed
+
+.PHONY: zstd
+zstd:
+	$(MAKE) -C $(PRGDIR) zstd
+	ln -sf $(PRGDIR)/zstd gzip
+	@echo PATH=$(PATH)
+	gzip --version
+
+.PHONY: clean
+clean:
+	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(RM) *.trs *.log
+	@echo Cleaning completed
+
+
+#------------------------------------------------------------------------------
+# validated only for Linux, macOS, Hurd and some BSD targets
+#------------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+
+test-%: zstd
+	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
+	# || echo ignoring error
+
+endif

externals/zstd/tests/regression/Makefile

@@ -0,0 +1,60 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+CFLAGS ?= -O3
+
+CURL_CFLAGS := $(shell curl-config --cflags)
+CURL_LDFLAGS := $(shell curl-config --libs) -pthread
+
+PROGDIR := ../../programs
+LIBDIR := ../../lib
+ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common -Wno-deprecated-declarations
+
+REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS)
+REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS)
+REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS)
+
+all: test
+
+xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+config.o: config.c config.h levels.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+method.h: data.h config.h result.h
+
+method.o: method.c method.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+result.o: result.c result.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+test.o: test.c data.h config.h method.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+.PHONY: libzstd.a
+libzstd.a:
+	$(MAKE) -C $(LIBDIR) libzstd.a-mt
+	cp $(LIBDIR)/libzstd.a .
+
+test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a
+	$(CC) $^ $(REGRESSION_LDFLAGS) -o $@
+
+.PHONY: clean
+clean:
+	$(MAKE) -C $(LIBDIR) clean
+	$(RM) *.o *.a test

externals/zstd/zlibWrapper/Makefile

@@ -0,0 +1,120 @@
+# Makefile for example of using zstd wrapper for zlib
+#
+# make - compiles examples
+# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
+# make test - runs examples
+
+
+# Paths to static and dynamic zlib and zstd libraries
+# Use "make ZLIB_PATH=path/to/zlib ZLIB_LIBRARY=path/to/libz.so" to select a path to library
+ZLIB_LIBRARY ?= -lz
+ZLIB_PATH ?= .
+
+ZSTDLIBDIR = ../lib
+ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
+ZLIBWRAPPER_PATH = .
+GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
+EXAMPLE_PATH = examples
+PROGRAMS_PATH = ../programs
+TEST_FILE = ../doc/zstd_compression_format.md
+
+vpath %.c $(PROGRAMS_PATH) $(EXAMPLE_PATH) $(ZLIBWRAPPER_PATH)
+
+
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH)       \
+            -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
+STDFLAGS  = -std=c89 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \
+            -DNO_snprintf -DNO_vsnprintf  # strict ANSI C89 is missing these prototypes
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
+            -Wdeclaration-after-statement -Wstrict-prototypes -Wundef     \
+            -Wstrict-aliasing=1
+CFLAGS   ?= -O3
+CFLAGS   += $(STDFLAGS) $(DEBUGFLAGS)
+CPPFLAGS += $(MOREFLAGS)
+LDLIBS   += $(ZLIB_LIBRARY)
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+default : release
+
+release : STDFLAGS =
+release : DEBUGFLAGS =
+release : all
+
+all: fitblk example zwrapbench minigzip
+
+test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd
+	./example
+	./example_zstd
+	./fitblk 10240 <$(TEST_FILE)
+	./fitblk 40960 <$(TEST_FILE)
+	./fitblk_zstd 10240 <$(TEST_FILE)
+	./fitblk_zstd 40960 <$(TEST_FILE)
+	@echo ---- minigzip start ----
+	./minigzip_zstd example$(EXT)
+	#cp example$(EXT).gz example$(EXT)_zstd.gz
+	./minigzip_zstd -d example$(EXT).gz
+	./minigzip example$(EXT)
+	#cp example$(EXT).gz example$(EXT)_gz.gz
+	./minigzip_zstd -d example$(EXT).gz
+	@echo ---- minigzip end ----
+	./zwrapbench -qi1b3B1K $(TEST_FILE)
+	./zwrapbench -rqi1b1e3 ../lib
+
+.PHONY: test-valgrind
+#test-valgrind: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
+test-valgrind: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
+test-valgrind: clean example fitblk example_zstd fitblk_zstd zwrapbench
+	@echo "\n ---- valgrind tests ----"
+	$(VALGRIND) ./example
+	$(VALGRIND) ./example_zstd
+	$(VALGRIND) ./fitblk 10240 <$(TEST_FILE)
+	$(VALGRIND) ./fitblk 40960 <$(TEST_FILE)
+	$(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE)
+	$(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE)
+	$(VALGRIND) ./zwrapbench -qi1b3B1K $(TEST_FILE)
+	$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
+
+#.c.o:
+#	$(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
+
+minigzip: minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
+
+minigzip_zstd: minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
+	$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
+
+example: example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
+
+example_zstd: example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
+	$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
+
+fitblk: fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
+
+fitblk_zstd: fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
+	$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
+
+zwrapbench: zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY)
+
+
+zstd_zlibwrapper.o: zstd_zlibwrapper.h
+
+zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
+zstdTurnedOn_zlibwrapper.o: zstd_zlibwrapper.c zstd_zlibwrapper.h
+	$(COMPILE.c) $< $(OUTPUT_OPTION)
+
+
+$(ZSTDLIBRARY):
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
+
+$(ZSTDLIBDIR)/libzstd.so:
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd
+
+
+clean:
+	-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o foo.gz example$(EXT) example_zstd$(EXT) fitblk$(EXT) fitblk_zstd$(EXT) zwrapbench$(EXT) minigzip$(EXT) minigzip_zstd$(EXT)
+	@echo Cleaning completed