First Commit

externals/zstd/tests/.gitignore

@@ -0,0 +1,69 @@
+# local binary (Makefile)
+fullbench
+fullbench32
+fullbench-lib
+fuzzer
+fuzzer32
+fuzzer-dll
+zbufftest
+zbufftest32
+zbufftest-dll
+zstreamtest
+zstreamtest32
+zstreamtest_asan
+zstreamtest_tsan
+zstreamtest_ubsan
+zstreamtest-dll
+datagen
+paramgrill
+paramgrill32
+roundTripCrash
+longmatch
+symbols
+legacy
+decodecorpus
+pool
+poolTests
+invalidDictionaries
+checkTag
+zcat
+zstdcat
+tm
+
+# test artifacts
+dictionary
+grillResults.txt
+_*
+tmp*
+*.zst
+*.gz
+!gzip/hufts-segv.gz
+result
+out
+*.zstd
+hello*
+world
+
+# Tmp test directory
+zstdtest
+speedTest
+versionsTest
+namespaceTest
+dirTest*
+
+# fuzzer
+afl
+
+# Local script
+startSpeedTest
+speedTest.pid
+*.bat
+
+# Generic Object files
+*.o
+*.ko
+
+# Generic Executables
+*.exe
+*.out
+*.app

externals/zstd/tests/DEPRECATED-test-zstd-speed.py

@@ -0,0 +1,378 @@
+#! /usr/bin/env python3
+# THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
+
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ##########################################################################
+
+# Limitations:
+# - doesn't support filenames with spaces
+# - dir1/zstd and dir2/zstd will be merged in a single results file
+
+import argparse
+import os           # getloadavg
+import string
+import subprocess
+import time         # strftime
+import traceback
+import hashlib
+import platform     # system
+
+script_version = 'v1.1.2 (2017-03-26)'
+default_repo_url = 'https://github.com/facebook/zstd.git'
+working_dir_name = 'speedTest'
+working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest
+clone_path = working_path + '/' + 'zstd'                # /path/to/zstd/tests/speedTest/zstd
+email_header = 'ZSTD_speedTest'
+pid = str(os.getpid())
+verbose = False
+clang_version = "unknown"
+gcc_version = "unknown"
+args = None
+
+
+def hashfile(hasher, fname, blocksize=65536):
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(blocksize), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def log(text):
+    print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
+
+
+def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
+    if print_command:
+        log("> " + command)
+    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
+    stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
+    stderr_lines = stderr_lines.decode("utf-8")
+    stdout_lines = stdout_lines.decode("utf-8")
+    if print_output:
+        if stdout_lines:
+            print(stdout_lines)
+        if stderr_lines:
+            print(stderr_lines)
+    if popen.returncode is not None and popen.returncode != 0:
+        if stderr_lines and not print_output and print_error:
+            print(stderr_lines)
+        raise RuntimeError(stdout_lines + stderr_lines)
+    return (stdout_lines + stderr_lines).splitlines()
+execute.cwd = None
+
+
+def does_command_exist(command):
+    try:
+        execute(command, verbose, False, False)
+    except Exception:
+        return False
+    return True
+
+
+def send_email(emails, topic, text, have_mutt, have_mail):
+    logFileName = working_path + '/' + 'tmpEmailContent'
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        if have_mutt:
+            execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
+        elif have_mail:
+            execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def send_email_with_attachments(branch, commit, last_commit, args, text, results_files,
+                                logFileName, have_mutt, have_mail):
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
+                      % (email_header, pid, branch, commit, last_commit,
+                         args.lowerLimit, args.ratioLimit)
+        if have_mutt:
+            execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files
+                    + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def git_get_branches():
+    execute('git fetch -p', verbose)
+    branches = execute('git branch -rl', verbose)
+    output = []
+    for line in branches:
+        if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line):
+            output.append(line.strip())
+    return output
+
+
+def git_get_changes(branch, commit, last_commit):
+    fmt = '--format="%h: (%an) %s, %ar"'
+    if last_commit is None:
+        commits = execute('git log -n 10 %s %s' % (fmt, commit))
+    else:
+        commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
+    return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
+
+
+def get_last_results(resultsFileName):
+    if not os.path.isfile(resultsFileName):
+        return None, None, None, None
+    commit = None
+    csize = []
+    cspeed = []
+    dspeed = []
+    with open(resultsFileName, 'r') as f:
+        for line in f:
+            words = line.split()
+            if len(words) <= 4:   # branch + commit + compilerVer + md5
+                commit = words[1]
+                csize = []
+                cspeed = []
+                dspeed = []
+            if (len(words) == 8) or (len(words) == 9):  # results: "filename" or "XX files"
+                csize.append(int(words[1]))
+                cspeed.append(float(words[3]))
+                dspeed.append(float(words[5]))
+    return commit, csize, cspeed, dspeed
+
+
+def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName,
+                          testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
+    sleepTime = 30
+    while os.getloadavg()[0] > args.maxLoadAvg:
+        log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds"
+            % (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
+        time.sleep(sleepTime)
+    start_load = str(os.getloadavg())
+    osType = platform.system()
+    if osType == 'Linux':
+        cpuSelector = "taskset --cpu-list 0"
+    else:
+        cpuSelector = ""
+    if args.dictionary:
+        result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
+    else:
+        result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True)
+    end_load = str(os.getloadavg())
+    linesExpected = args.lastCLevel + 1
+    if len(result) != linesExpected:
+        raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
+    with open(resultsFileName, "a") as myfile:
+        myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum))
+        myfile.write('\n'.join(result) + '\n')
+        myfile.close()
+        if (last_cspeed == None):
+            log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
+            return ""
+        commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
+        text = ""
+        for i in range(0, min(len(cspeed), len(last_cspeed))):
+            print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName))
+            if (cspeed[i]/last_cspeed[i] < args.lowerLimit):
+                text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
+            if (dspeed[i]/last_dspeed[i] < args.lowerLimit):
+                text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
+            if (float(last_csize[i])/csize[i] < args.ratioLimit):
+                text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
+        if text:
+            text = args.message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s\n%s  last_commit=%s  md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text
+        return text
+
+
+def update_config_file(branch, commit):
+    last_commit = None
+    commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
+    if os.path.isfile(commitFileName):
+        with open(commitFileName, 'r') as infile:
+            last_commit = infile.read()
+    with open(commitFileName, 'w') as outfile:
+        outfile.write(commit)
+    return last_commit
+
+
+def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName):
+    last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
+    if not args.dry_run:
+        text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
+        if text:
+            log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
+            text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
+    return text
+
+
+def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
+    local_branch = branch.split('/')[1]
+    version = local_branch.rpartition('-')[2] + '_' + commit
+    if not args.dry_run:
+        execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
+                'mv programs/zstd programs/zstd_clang && ' +
+                'make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
+    md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd')
+    md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32')
+    md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang')
+    print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
+    print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version))
+
+    logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
+    text_to_send = []
+    results_files = ""
+    if args.dictionary:
+        dictName = args.dictionary.rpartition('/')[2]
+    else:
+        dictName = None
+
+    for filePath in testFilePaths:
+        fileName = filePath.rpartition('/')[2]
+        if dictName:
+            resultsFileName = working_path + "/" + dictName.replace(".", "_") + "_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        else:
+            resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        text = double_check(branch, commit, args, 'zstd', md5_zstd, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
+        if text:
+            text_to_send.append(text)
+            results_files += resultsFileName + " "
+        resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        text = double_check(branch, commit, args, 'zstd32', md5_zstd32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
+        if text:
+            text_to_send.append(text)
+            results_files += resultsFileName + " "
+        resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName)
+        if text:
+            text_to_send.append(text)
+            results_files += resultsFileName + " "
+    if text_to_send:
+        send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('testFileNames', help='file or directory names list for speed benchmark')
+    parser.add_argument('emails', help='list of e-mail addresses to send warnings')
+    parser.add_argument('--dictionary', '-D', help='path to the dictionary')
+    parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="")
+    parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
+    parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98)
+    parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999)
+    parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
+    parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
+    parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
+    parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800)
+    parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
+    parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
+    args = parser.parse_args()
+    verbose = args.verbose
+
+    # check if test files are accessible
+    testFileNames = args.testFileNames.split()
+    testFilePaths = []
+    for fileName in testFileNames:
+        fileName = os.path.expanduser(fileName)
+        if os.path.isfile(fileName) or os.path.isdir(fileName):
+            testFilePaths.append(os.path.abspath(fileName))
+        else:
+            log("ERROR: File/directory not found: " + fileName)
+            exit(1)
+
+    # check if dictionary is accessible
+    if args.dictionary:
+        args.dictionary = os.path.abspath(os.path.expanduser(args.dictionary))
+        if not os.path.isfile(args.dictionary):
+            log("ERROR: Dictionary not found: " + args.dictionary)
+            exit(1)
+
+    # check availability of e-mail senders
+    have_mutt = does_command_exist("mutt -h")
+    have_mail = does_command_exist("mail -V")
+    if not have_mutt and not have_mail:
+        log("ERROR: e-mail senders 'mail' or 'mutt' not found")
+        exit(1)
+
+    clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
+    gcc_version = execute("gcc -dumpversion", verbose)[0];
+
+    if verbose:
+        print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
+        print("working_path=%s" % working_path)
+        print("clone_path=%s" % clone_path)
+        print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
+        print("message=%s" % args.message)
+        print("emails=%s" % args.emails)
+        print("dictionary=%s" % args.dictionary)
+        print("maxLoadAvg=%s" % args.maxLoadAvg)
+        print("lowerLimit=%s" % args.lowerLimit)
+        print("ratioLimit=%s" % args.ratioLimit)
+        print("lastCLevel=%s" % args.lastCLevel)
+        print("sleepTime=%s" % args.sleepTime)
+        print("timeout=%s" % args.timeout)
+        print("dry_run=%s" % args.dry_run)
+        print("verbose=%s" % args.verbose)
+        print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
+
+    # clone ZSTD repo if needed
+    if not os.path.isdir(working_path):
+        os.mkdir(working_path)
+    if not os.path.isdir(clone_path):
+        execute.cwd = working_path
+        execute('git clone ' + args.repoURL)
+    if not os.path.isdir(clone_path):
+        log("ERROR: ZSTD clone not found: " + clone_path)
+        exit(1)
+    execute.cwd = clone_path
+
+    # check if speedTest.pid already exists
+    pidfile = "./speedTest.pid"
+    if os.path.isfile(pidfile):
+        log("ERROR: %s already exists, exiting" % pidfile)
+        exit(1)
+
+    send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
+    with open(pidfile, 'w') as the_file:
+        the_file.write(pid)
+
+    branch = ""
+    commit = ""
+    first_time = True
+    while True:
+        try:
+            if first_time:
+                first_time = False
+            else:
+                time.sleep(args.sleepTime)
+            loadavg = os.getloadavg()[0]
+            if (loadavg <= args.maxLoadAvg):
+                branches = git_get_branches()
+                for branch in branches:
+                    commit = execute('git show -s --format=%h ' + branch, verbose)[0]
+                    last_commit = update_config_file(branch, commit)
+                    if commit == last_commit:
+                        log("skipping branch %s: head %s already processed" % (branch, commit))
+                    else:
+                        log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
+                        execute('git checkout -- . && git checkout ' + branch)
+                        print(git_get_changes(branch, commit, last_commit))
+                        test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
+            else:
+                log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
+            if verbose:
+                log("sleep for %s seconds" % args.sleepTime)
+        except Exception as e:
+            stack = traceback.format_exc()
+            email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
+            send_email(args.emails, email_topic, stack, have_mutt, have_mail)
+            print(stack)
+        except KeyboardInterrupt:
+            os.unlink(pidfile)
+            send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
+            exit(0)

externals/zstd/tests/README.md

@@ -0,0 +1,184 @@
+Programs and scripts for automated testing of Zstandard
+=======================================================
+
+This directory contains the following programs and scripts:
+- `datagen` : Synthetic and parametrable data generator, for tests
+- `fullbench`  : Precisely measure speed for each zstd inner functions
+- `fuzzer`  : Test tool, to check zstd integrity on target platform
+- `paramgrill` : parameter tester for zstd
+- `test-zstd-speed.py` : script for testing zstd speed difference between commits
+- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
+- `zstreamtest` : Fuzzer test tool for zstd streaming API
+- `legacy` : Test tool to test decoding of legacy zstd frames
+- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
+
+
+#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
+
+This script creates `versionsTest` directory to which zstd repository is cloned.
+Then all tagged (released) versions of zstd are compiled.
+In the following step interoperability between zstd versions is checked.
+
+#### `automated-benchmarking.py` - script for benchmarking zstd prs to dev
+
+This script benchmarks facebook:dev and changes from pull requests made to zstd and compares
+them against facebook:dev to detect regressions. This script currently runs on a dedicated
+desktop machine for every pull request that is made to the zstd repo but can also
+be run on any machine via the command line interface.
+
+There are three modes of usage for this script: fastmode will just run a minimal single
+build comparison (between facebook:dev and facebook:release), onetime will pull all the current
+pull requests from the zstd repo and compare facebook:dev to all of them once, continuous
+will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev.
+
+```
+Example usage: python automated_benchmarking.py
+```
+
+```
+usage: automated_benchmarking.py [-h] [--directory DIRECTORY]
+                                 [--levels LEVELS] [--iterations ITERATIONS]
+                                 [--emails EMAILS] [--frequency FREQUENCY]
+                                 [--mode MODE] [--dict DICT]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --directory DIRECTORY
+                        directory with files to benchmark
+  --levels LEVELS       levels to test e.g. ('1,2,3')
+  --iterations ITERATIONS
+                        number of benchmark iterations to run
+  --emails EMAILS       email addresses of people who will be alerted upon
+                        regression. Only for continuous mode
+  --frequency FREQUENCY
+                        specifies the number of seconds to wait before each
+                        successive check for new PRs in continuous mode
+  --mode MODE           'fastmode', 'onetime', 'current', or 'continuous' (see
+                        README.md for details)
+  --dict DICT           filename of dictionary to use (when set, this
+                        dictionary will be used to compress the files provided
+                        inside --directory)
+```
+
+#### `test-zstd-speed.py` - script for testing zstd speed difference between commits
+
+DEPRECATED
+
+This script creates `speedTest` directory to which zstd repository is cloned.
+Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter).
+After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits.
+If a new commit is found it is compiled and a speed benchmark for this commit is performed.
+The results of the speed benchmark are compared to the previous results.
+If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
+If second results are also lower than `lowerLimit` the warning e-mail is sent to recipients from the list (the `emails` parameter).
+
+Additional remarks:
+- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
+- Using the script with virtual machines can lead to large variations of speed results
+- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75)
+- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning
+
+
+The example usage with two test files, one e-mail address, and with an additional message:
+```
+./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
+```
+
+To run the script in background please use:
+```
+nohup ./test-zstd-speed.py testFileNames emails &
+```
+
+The full list of parameters:
+```
+positional arguments:
+  testFileNames         file names list for speed benchmark
+  emails                list of e-mail addresses to send warnings
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --message MESSAGE     attach an additional message to e-mail
+  --lowerLimit LOWERLIMIT
+                        send email if speed is lower than given limit
+  --maxLoadAvg MAXLOADAVG
+                        maximum load average to start testing
+  --lastCLevel LASTCLEVEL
+                        last compression level for testing
+  --sleepTime SLEEPTIME
+                        frequency of repository checking in seconds
+```
+
+#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
+Command line tool to generate test .zst files.
+
+This tool will generate .zst files with checksums,
+as well as optionally output the corresponding correct uncompressed data for
+extra verification.
+
+Example:
+```
+./decodecorpus -ptestfiles -otestfiles -n10000 -s5
+```
+will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
+with the zstd checksum field set,
+as well as the 10,000 original files for more detailed comparison of decompression results.
+
+```
+./decodecorpus -t -T1mn
+```
+will choose a random seed, and for 1 minute,
+generate random test frames and ensure that the
+zstd library correctly decompresses them in both simple and streaming modes.
+
+#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints
+
+Full list of arguments
+```
+ -T#          : set level 1 speed objective
+ -B#          : cut input into blocks of size # (default : single block)
+ -S           : benchmarks a single run (example command: -Sl3w10h12)
+    w# - windowLog
+    h# - hashLog
+    c# - chainLog
+    s# - searchLog
+    l# - minMatch
+    t# - targetLength
+    S# - strategy
+    L# - level
+ --zstd=      : Single run, parameter selection syntax same as zstdcli with more parameters
+                    (Added forceAttachDictionary / fadt)
+                    When invoked with --optimize, this represents the sample to exceed.
+ --optimize=  : find parameters to maximize compression ratio given parameters
+                    Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
+    cSpeed=   : Minimum compression speed
+    dSpeed=   : Minimum decompression speed
+    cMem=     : Maximum compression memory
+    lvl=      : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
+    stc=      : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
+              : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
+                    (Lower value will begin with stronger strategies) (Default 90%)
+    speedRatio=   (accepts decimals)
+              : determines value of gains in speed vs gains in ratio
+                    when determining overall winner (default 5 (1% ratio = 5% speed)).
+    tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
+                    Higher values will make optimizer run longer, more chances to find better solution.
+    memLog    : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
+                    Setting memLog = 0 turns off memoization
+ --display=   : specify which parameters are included in the output
+                    can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
+                    (Default: display all params available)
+ -P#          : generated sample compressibility (when no file is provided)
+ -t#          : Caps runtime of operation in seconds (default: 99999 seconds (about 27 hours))
+ -v           : Prints Benchmarking output
+ -D           : Next argument dictionary file
+ -s           : Benchmark all files separately
+ -q           : Quiet, repeat for more quiet
+                  -q Prints parameters + results whenever a new best is found
+                  -qq Only prints parameters whenever a new best is found, prints final parameters + results
+                  -qqq Only print final parameters + results
+                  -qqqq Only prints final parameter set in the form --zstd=
+ -v           : Verbose, cancels quiet, repeat for more volume
+                  -v Prints all candidate parameters and results
+
+```
+ Any inputs afterwards are treated as files to benchmark.

externals/zstd/tests/automated_benchmarking.py

@@ -0,0 +1,326 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ##########################################################################
+
+import argparse
+import glob
+import json
+import os
+import time
+import pickle as pk
+import subprocess
+import urllib.request
+
+
+GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open"
+GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd"
+RELEASE_BUILD = {"user": "facebook", "branch": "dev", "hash": None}
+
+# check to see if there are any new PRs every minute
+DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60
+PREVIOUS_PRS_FILENAME = "prev_prs.pk"
+
+# Not sure what the threshold for triggering alarms should be
+# 1% regression sounds like a little too sensitive but the desktop
+# that I'm running it on is pretty stable so I think this is fine
+CSPEED_REGRESSION_TOLERANCE = 0.01
+DSPEED_REGRESSION_TOLERANCE = 0.01
+
+
+def get_new_open_pr_builds(prev_state=True):
+    prev_prs = None
+    if os.path.exists(PREVIOUS_PRS_FILENAME):
+        with open(PREVIOUS_PRS_FILENAME, "rb") as f:
+            prev_prs = pk.load(f)
+    data = json.loads(urllib.request.urlopen(GITHUB_API_PR_URL).read().decode("utf-8"))
+    prs = {
+        d["url"]: {
+            "user": d["user"]["login"],
+            "branch": d["head"]["ref"],
+            "hash": d["head"]["sha"].strip(),
+        }
+        for d in data
+    }
+    with open(PREVIOUS_PRS_FILENAME, "wb") as f:
+        pk.dump(prs, f)
+    if not prev_state or prev_prs == None:
+        return list(prs.values())
+    return [pr for url, pr in prs.items() if url not in prev_prs or prev_prs[url] != pr]
+
+
+def get_latest_hashes():
+    tmp = subprocess.run(["git", "log", "-1"], stdout=subprocess.PIPE).stdout.decode(
+        "utf-8"
+    )
+    sha1 = tmp.split("\n")[0].split(" ")[1]
+    tmp = subprocess.run(
+        ["git", "show", "{}^1".format(sha1)], stdout=subprocess.PIPE
+    ).stdout.decode("utf-8")
+    sha2 = tmp.split("\n")[0].split(" ")[1]
+    tmp = subprocess.run(
+        ["git", "show", "{}^2".format(sha1)], stdout=subprocess.PIPE
+    ).stdout.decode("utf-8")
+    sha3 = "" if len(tmp) == 0 else tmp.split("\n")[0].split(" ")[1]
+    return [sha1.strip(), sha2.strip(), sha3.strip()]
+
+
+def get_builds_for_latest_hash():
+    hashes = get_latest_hashes()
+    for b in get_new_open_pr_builds(False):
+        if b["hash"] in hashes:
+            return [b]
+    return []
+
+
+def clone_and_build(build):
+    if build["user"] != None:
+        github_url = GITHUB_URL_TEMPLATE.format(build["user"])
+        os.system(
+            """
+            rm -rf zstd-{user}-{sha} &&
+            git clone {github_url} zstd-{user}-{sha} &&
+            cd zstd-{user}-{sha} &&
+            {checkout_command}
+            make -j &&
+            cd ../
+        """.format(
+                user=build["user"],
+                github_url=github_url,
+                sha=build["hash"],
+                checkout_command="git checkout {} &&".format(build["hash"])
+                if build["hash"] != None
+                else "",
+            )
+        )
+        return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"])
+    else:
+        os.system("cd ../ && make -j && cd tests")
+        return "../zstd"
+
+
+def parse_benchmark_output(output):
+    idx = [i for i, d in enumerate(output) if d == "MB/s"]
+    return [float(output[idx[0] - 1]), float(output[idx[1] - 1])]
+
+
+def benchmark_single(executable, level, filename):
+    return parse_benchmark_output((
+        subprocess.run(
+            [executable, "-qb{}".format(level), filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        )
+        .stdout.decode("utf-8")
+        .split(" ")
+    ))
+
+
+def benchmark_n(executable, level, filename, n):
+    speeds_arr = [benchmark_single(executable, level, filename) for _ in range(n)]
+    cspeed, dspeed = max(b[0] for b in speeds_arr), max(b[1] for b in speeds_arr)
+    print(
+        "Bench (executable={} level={} filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format(
+            os.path.basename(executable),
+            level,
+            os.path.basename(filename),
+            n,
+            cspeed,
+            dspeed,
+        )
+    )
+    return (cspeed, dspeed)
+
+
+def benchmark(build, filenames, levels, iterations):
+    executable = clone_and_build(build)
+    return [
+        [benchmark_n(executable, l, f, iterations) for f in filenames] for l in levels
+    ]
+
+
+def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations):
+    cspeeds, dspeeds = [], []
+    for _ in range(iterations):
+        output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stdout=subprocess.PIPE).stdout.decode("utf-8").split(" ")
+        cspeed, dspeed = parse_benchmark_output(output)
+        cspeeds.append(cspeed)
+        dspeeds.append(dspeed)
+    max_cspeed, max_dspeed = max(cspeeds), max(dspeeds)
+    print(
+        "Bench (executable={} level={} filenames_directory={}, dictionary_filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format(
+            os.path.basename(executable),
+            level,
+            os.path.basename(filenames_directory),
+            os.path.basename(dictionary_filename),
+            iterations,
+            max_cspeed,
+            max_dspeed,
+        )
+    )
+    return (max_cspeed, max_dspeed)
+
+
+def benchmark_dictionary(build, filenames_directory, dictionary_filename, levels, iterations):
+    executable = clone_and_build(build)
+    return [benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, l, iterations) for l in levels]
+
+
+def parse_regressions_and_labels(old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build):
+    cspeed_reg = (old_cspeed - new_cspeed) / old_cspeed
+    dspeed_reg = (old_dspeed - new_dspeed) / old_dspeed
+    baseline_label = "{}:{} ({})".format(
+        baseline_build["user"], baseline_build["branch"], baseline_build["hash"]
+    )
+    test_label = "{}:{} ({})".format(
+        test_build["user"], test_build["branch"], test_build["hash"]
+    )
+    return cspeed_reg, dspeed_reg, baseline_label, test_label
+
+
+def get_regressions(baseline_build, test_build, iterations, filenames, levels):
+    old = benchmark(baseline_build, filenames, levels, iterations)
+    new = benchmark(test_build, filenames, levels, iterations)
+    regressions = []
+    for j, level in enumerate(levels):
+        for k, filename in enumerate(filenames):
+            old_cspeed, old_dspeed = old[j][k]
+            new_cspeed, new_dspeed = new[j][k]
+            cspeed_reg, dspeed_reg, baseline_label, test_label = parse_regressions_and_labels(
+                old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
+            )
+            if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
+                regressions.append(
+                    "[COMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
+                        level,
+                        filename,
+                        baseline_label,
+                        test_label,
+                        old_cspeed,
+                        new_cspeed,
+                        cspeed_reg * 100.0,
+                    )
+                )
+            if dspeed_reg > DSPEED_REGRESSION_TOLERANCE:
+                regressions.append(
+                    "[DECOMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
+                        level,
+                        filename,
+                        baseline_label,
+                        test_label,
+                        old_dspeed,
+                        new_dspeed,
+                        dspeed_reg * 100.0,
+                    )
+                )
+    return regressions
+
+def get_regressions_dictionary(baseline_build, test_build, filenames_directory, dictionary_filename, levels, iterations):
+    old = benchmark_dictionary(baseline_build, filenames_directory, dictionary_filename, levels, iterations)
+    new = benchmark_dictionary(test_build, filenames_directory, dictionary_filename, levels, iterations)
+    regressions = []
+    for j, level in enumerate(levels):
+        old_cspeed, old_dspeed = old[j]
+        new_cspeed, new_dspeed = new[j]
+        cspeed_reg, dspeed_reg, baesline_label, test_label = parse_regressions_and_labels(
+            old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
+        )
+        if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
+            regressions.append(
+                "[COMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
+                    level,
+                    filenames_directory,
+                    dictionary_filename,
+                    baseline_label,
+                    test_label,
+                    old_cspeed,
+                    new_cspeed,
+                    cspeed_reg * 100.0,
+                )
+            )
+        if dspeed_reg > DSPEED_REGRESSION_TOLERANCE:
+            regressions.append(
+                "[DECOMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
+                    level,
+                    filenames_directory,
+                    dictionary_filename,
+                    baseline_label,
+                    test_label,
+                    old_dspeed,
+                    new_dspeed,
+                    dspeed_reg * 100.0,
+                )
+            )
+        return regressions
+
+
+def main(filenames, levels, iterations, builds=None, emails=None, continuous=False, frequency=DEFAULT_MAX_API_CALL_FREQUENCY_SEC, dictionary_filename=None):
+    if builds == None:
+        builds = get_new_open_pr_builds()
+    while True:
+        for test_build in builds:
+            if dictionary_filename == None:
+                regressions = get_regressions(
+                    RELEASE_BUILD, test_build, iterations, filenames, levels
+                )
+            else:
+                regressions = get_regressions_dictionary(
+                    RELEASE_BUILD, test_build, filenames, dictionary_filename, levels, iterations
+                )
+            body = "\n".join(regressions)
+            if len(regressions) > 0:
+                if emails != None:
+                    os.system(
+                        """
+                        echo "{}" | mutt -s "[zstd regression] caused by new pr" {}
+                    """.format(
+                            body, emails
+                        )
+                    )
+                    print("Emails sent to {}".format(emails))
+                print(body)
+        if not continuous:
+            break
+        time.sleep(frequency)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--directory", help="directory with files to benchmark", default="golden-compression")
+    parser.add_argument("--levels", help="levels to test e.g. ('1,2,3')", default="1")
+    parser.add_argument("--iterations", help="number of benchmark iterations to run", default="1")
+    parser.add_argument("--emails", help="email addresses of people who will be alerted upon regression. Only for continuous mode", default=None)
+    parser.add_argument("--frequency", help="specifies the number of seconds to wait before each successive check for new PRs in continuous mode", default=DEFAULT_MAX_API_CALL_FREQUENCY_SEC)
+    parser.add_argument("--mode", help="'fastmode', 'onetime', 'current', or 'continuous' (see README.md for details)", default="current")
+    parser.add_argument("--dict", help="filename of dictionary to use (when set, this dictionary will be used to compress the files provided inside --directory)", default=None)
+
+    args = parser.parse_args()
+    filenames = args.directory
+    levels = [int(l) for l in args.levels.split(",")]
+    mode = args.mode
+    iterations = int(args.iterations)
+    emails = args.emails
+    frequency = int(args.frequency)
+    dictionary_filename = args.dict
+
+    if dictionary_filename == None:
+        filenames = glob.glob("{}/**".format(filenames))
+
+    if (len(filenames) == 0):
+        print("0 files found")
+        quit()
+
+    if mode == "onetime":
+        main(filenames, levels, iterations, frequency=frequenc, dictionary_filename=dictionary_filename)
+    elif mode == "current":
+        builds = [{"user": None, "branch": "None", "hash": None}]
+        main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
+    elif mode == "fastmode":
+        builds = [{"user": "facebook", "branch": "release", "hash": None}]
+        main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
+    else:
+        main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename)

externals/zstd/tests/bigdict.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include "datagen.h"
+#include "mem.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+static int
+compress(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+         void* dst, size_t dstCapacity,
+         void const* src, size_t srcSize,
+         void* roundtrip, ZSTD_EndDirective end)
+{
+    ZSTD_inBuffer in = {src, srcSize, 0};
+    ZSTD_outBuffer out = {dst, dstCapacity, 0};
+    int ended = 0;
+
+    while (!ended && (in.pos < in.size || out.pos > 0)) {
+        size_t rc;
+        out.pos = 0;
+        rc = ZSTD_compressStream2(cctx, &out, &in, end);
+        if (ZSTD_isError(rc))
+            return 1;
+        if (end == ZSTD_e_end && rc == 0)
+            ended = 1;
+        {
+            ZSTD_inBuffer rtIn = {dst, out.pos, 0};
+            ZSTD_outBuffer rtOut = {roundtrip, srcSize, 0};
+            rc = 1;
+            while (rtIn.pos < rtIn.size || rtOut.pos > 0) {
+                rtOut.pos = 0;
+                rc = ZSTD_decompressStream(dctx, &rtOut, &rtIn);
+                if (ZSTD_isError(rc)) {
+                    fprintf(stderr, "Decompression error: %s\n", ZSTD_getErrorName(rc));
+                    return 1;
+                }
+                if (rc == 0)
+                    break;
+            }
+            if (ended && rc != 0) {
+                fprintf(stderr, "Frame not finished!\n");
+                return 1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int main(int argc, const char** argv)
+{
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    const size_t dataSize = (size_t)1 << 30;
+    const size_t outSize = ZSTD_compressBound(dataSize);
+    const size_t bufferSize = (size_t)1 << 31;
+    char* buffer = (char*)malloc(bufferSize);
+    void* out = malloc(outSize);
+    void* roundtrip = malloc(dataSize);
+    (void)argc;
+    (void)argv;
+
+    if (!buffer || !out || !roundtrip || !cctx || !dctx) {
+        fprintf(stderr, "Allocation failure\n");
+        return 1;
+    }
+
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10)))
+        return 1;
+    if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10)))
+        return 1;
+
+    if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31)))
+        return 1;
+
+    RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe);
+
+    /* Compress 30 GB */
+    {
+        int i;
+        for (i = 0; i < 10; ++i) {
+            fprintf(stderr, "Compressing 1 GB\n");
+            if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue))
+                return 1;
+        }
+    }
+    fprintf(stderr, "Compressing 1 GB\n");
+    if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end))
+        return 1;
+
+    fprintf(stderr, "Success!\n");
+
+    free(roundtrip);
+    free(out);
+    free(buffer);
+    ZSTD_freeDCtx(dctx);
+    ZSTD_freeCCtx(cctx);
+    return 0;
+}

externals/zstd/tests/checkTag.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* checkTag : validation tool for libzstd
+ * command :
+ * $ ./checkTag tag
+ * checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any}
+ * The tag is then compared to zstd version number.
+ * They are compatible if first 3 digits are identical.
+ * Anything beyond that is free, and doesn't impact validation.
+ * Example : tag v1.8.1.2 is compatible with version 1.8.1
+ * When tag and version are not compatible, program exits with error code 1.
+ * When they are compatible, it exists with a code 0.
+ * checkTag is intended to be used in automated testing environment.
+ */
+
+#include <stdio.h>   /* printf */
+#include <string.h>  /* strlen, strncmp */
+#include "zstd.h"    /* ZSTD_VERSION_STRING */
+
+
+/*  validate() :
+ * @return 1 if tag is compatible, 0 if not.
+ */
+static int validate(const char* const tag)
+{
+    size_t const tagLength = strlen(tag);
+    size_t const verLength = strlen(ZSTD_VERSION_STRING);
+
+    if (tagLength < 2) return 0;
+    if (tag[0] != 'v') return 0;
+    if (tagLength <= verLength) return 0;
+
+    if (strncmp(ZSTD_VERSION_STRING, tag+1, verLength)) return 0;
+
+    return 1;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    const char* const tag = argv[1];
+    if (argc!=2) {
+        printf("incorrect usage : %s tag \n", exeName);
+        return 2;
+    }
+
+    printf("Version : %s \n", ZSTD_VERSION_STRING);
+    printf("Tag     : %s \n", tag);
+
+    if (validate(tag)) {
+        printf("OK : tag is compatible with zstd version \n");
+        return 0;
+    }
+
+    printf("!! error : tag and versions are not compatible !! \n");
+    return 1;
+}

externals/zstd/tests/check_size.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+import os
+import subprocess
+import sys
+
+if len(sys.argv) != 3:
+	print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT")
+	sys.exit(1)
+
+file = sys.argv[1]
+limit = int(sys.argv[2])
+
+if not os.path.exists(file):
+	print(f"{file} does not exist")
+	sys.exit(1)
+
+size = os.path.getsize(file)
+
+if size > limit:
+	print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes")
+	sys.exit(1)

externals/zstd/tests/cli-tests/.gitignore

@@ -0,0 +1,6 @@
+!bin/
+!datagen
+!zstdcat
+
+scratch/
+bin/symlinks

externals/zstd/tests/cli-tests/README.md

@@ -0,0 +1,258 @@
+# CLI tests
+
+The CLI tests are focused on testing the zstd CLI.
+They are intended to be simple tests that the CLI and arguments work as advertised.
+They are not intended to test the library, only the code in `programs/`.
+The library will get incidental coverage, but if you find yourself trying to trigger a specific condition in the library, this is the wrong tool.
+
+## Test runner usage
+
+The test runner `run.py` will run tests against the in-tree build of `zstd` and `datagen` by default. Which means that `zstd` and `datagen` must be built.
+
+The `zstd` binary used can be passed with `--zstd /path/to/zstd`.
+Additionally, to run `zstd` through a tool like `valgrind` or `qemu`, set the `--exec-prefix 'valgrind -q'` flag.
+
+Similarly, the `--datagen`, and `--zstdgrep` flags can be set to specify
+the paths to their respective binaries. However, these tools do not use
+the `EXEC_PREFIX`.
+
+Each test executes in its own scratch directory under `scratch/test/name`. E.g. `scratch/basic/help.sh/`. Normally these directories are removed after the test executes. However, the `--preserve` flag will preserve these directories after execution, and save the tests exit code, stdout, and stderr in the scratch directory to `exit`, `stderr`, and `stdout` respectively. This can be useful for debugging/editing a test and updating the expected output.
+
+### Running all the tests
+
+By default the test runner `run.py` will run all the tests, and report the results.
+
+Examples:
+
+```
+./run.py
+./run.py --preserve
+./run.py --zstd ../../build/programs/zstd --datagen ../../build/tests/datagen
+```
+
+### Running specific tests
+
+A set of test names can be passed to the test runner `run.py` to only execute those tests.
+This can be useful for writing or debugging a test, especially with `--preserve`.
+
+The test name can either be the path to the test file, or the test name, which is the path relative to the test directory.
+
+Examples:
+
+```
+./run.py basic/help.sh
+./run.py --preserve basic/help.sh basic/version.sh
+./run.py --preserve --verbose basic/help.sh
+```
+
+### Updating exact output
+
+If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
+
+Example:
+```
+./run.py --set-exact-output
+./run.py basic/help.sh --set-exact-output
+```
+
+## Writing a test
+
+Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
+After the script executes, the exit code, stderr, and stdout are compared against the expectations.
+
+Each test is run in a clean directory that the test can use for intermediate files. This directory will be cleaned up at the end of the test, unless `--preserve` is passed to the test runner. Additionally, the `setup` script can prepare the directory before the test runs.
+
+### Calling zstd, utilities, and environment variables
+
+The `$PATH` for tests is prepended with the `bin/` sub-directory, which contains helper scripts for ease of testing.
+The `zstd` binary will call the zstd binary specified by `run.py` with the correct `$EXEC_PREFIX`.
+Similarly, `datagen`, `unzstd`, `zstdgrep`, `zstdcat`, etc, are provided.
+
+Helper utilities like `cmp_size`, `println`, and `die` are provided here too. See their scripts for details.
+
+Common shell script libraries are provided under `common/`, with helper variables and functions. They can be sourced with `source "$COMMON/library.sh`.
+
+Lastly, environment variables are provided for testing, which can be listed when calling `run.py` with `--verbose`.
+They are generally used by the helper scripts in `bin/` to coordinate everything.
+
+### Basic test case
+
+When executing your `$TEST` executable, by default the exit code is expected to be `0`. However, you can provide an alternate expected exit code in a `$TEST.exit` file.
+
+When executing your `$TEST` executable, by default the expected stderr and stdout are empty. However, you can override the default by providing one of three files:
+
+* `$TEST.{stdout,stderr}.exact`
+* `$TEST.{stdout,stderr}.glob`
+* `$TEST.{stdout,stderr}.ignore`
+
+If you provide a `.exact` file, the output is expected to exactly match, byte-for-byte.
+
+If you provide a `.glob` file, the output is expected to match the expected file, where each line is interpreted as a glob syntax. Additionally, a line containing only `...` matches all lines until the next expected line matches.
+
+If you provide a `.ignore` file, the output is ignored.
+
+#### Passing examples
+
+All these examples pass.
+
+Exit 1, and change the expectation to be 1.
+
+```
+exit-1.sh
+---
+#!/bin/sh
+exit 1
+---
+
+exit-1.sh.exit
+---
+1
+---
+```
+
+Check the stdout output exactly matches.
+
+```
+echo.sh
+---
+#!/bin/sh
+echo "hello world"
+---
+
+echo.sh.stdout.exact
+---
+hello world
+---
+```
+
+Check the stderr output using a glob.
+
+```
+random.sh
+---
+#!/bin/sh
+head -c 10 < /dev/urandom | xxd >&2
+---
+
+random.sh.stderr.glob
+---
+00000000: * * * * *                 *
+```
+
+Multiple lines can be matched with ...
+
+```
+random-num-lines.sh
+---
+#!/bin/sh
+echo hello
+seq 0 $RANDOM
+echo world
+---
+
+random-num-lines.sh.stdout.glob
+---
+hello
+0
+...
+world
+---
+```
+
+#### Failing examples
+
+Exit code is expected to be 0, but is 1.
+
+```
+exit-1.sh
+---
+#!/bin/sh
+exit 1
+---
+```
+
+Stdout is expected to be empty, but isn't.
+
+```
+echo.sh
+---
+#!/bin/sh
+echo hello world
+```
+
+Stderr is expected to be hello but is world.
+
+```
+hello.sh
+---
+#!/bin/sh
+echo world >&2
+---
+
+hello.sh.stderr.exact
+---
+hello
+---
+```
+
+### Setup & teardown scripts
+
+Finally, test writing can be eased with setup and teardown scripts.
+Each directory in the test directory is a test-suite consisting of all tests within that directory (but not sub-directories).
+This test suite can come with 4 scripts to help test writing:
+
+* `setup_once`
+* `teardown_once`
+* `setup`
+* `teardown`
+
+The `setup_once` and `teardown_once` are run once before and after all the tests in the suite respectively.
+They operate in the scratch directory for the test suite, which is the parent directory of each scratch directory for each test case.
+They can do work that is shared between tests to improve test efficiency.
+For example, the `dictionaries/setup_once` script builds several dictionaries, for use in the `dictionaries` tests.
+
+The `setup` and `teardown` scripts run before and after each test case respectively, in the test case's scratch directory.
+These scripts can do work that is shared between test cases to make tests more succinct.
+For example, the `dictionaries/setup` script copies the dictionaries built by the `dictionaries/setup_once` script into the test's scratch directory, to make them easier to use, and make sure they aren't accidentally modified.
+
+#### Examples
+
+```
+basic/setup
+---
+#!/bin/sh
+# Create some files for testing with
+datagen > file
+datagen > file0
+datagen > file1
+---
+
+basic/test.sh
+---
+#!/bin/sh
+zstd file file0 file1
+---
+
+dictionaries/setup_once
+---
+#!/bin/sh
+set -e
+
+mkdir files/ dicts/
+for i in $(seq 10); do
+	datagen -g1000 > files/$i
+done
+
+zstd --train -r files/ -o dicts/0
+---
+
+dictionaries/setup
+---
+#!/bin/sh
+
+# Runs in the test case's scratch directory.
+# The test suite's scratch directory that
+# `setup_once` operates in is the parent directory.
+cp -r ../files ../dicts .
+---
+```

externals/zstd/tests/cli-tests/basic/help.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+println "+ zstd -h"
+zstd -h
+println "+ zstd -H"
+zstd -H
+println "+ zstd --help"
+zstd --help

externals/zstd/tests/cli-tests/basic/help.sh.stdout.glob

@@ -0,0 +1,34 @@
++ zstd -h
+Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
+
+Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
+
+Options:
+  -o OUTPUT                     Write output to a single file, OUTPUT.
+  -k, --keep                    Preserve INPUT file(s). *Default*
+  --rm                          Remove INPUT file(s) after successful (de)compression.
+
+  -#                            Desired compression level, where `#` is a number between 1 and 19;
+                                lower numbers provide faster compression, higher numbers yield
+                                better compression ratios. *Default: 3*
+
+  -d, --decompress              Perform decompression.
+  -D DICT                       Use DICT as the dictionary for compression or decompression.
+
+  -f, --force                   Disable input and output checks. Allows overwriting existing files,
+                                receiving input from the console, printing output to STDOUT, and
+                                operating on links, block devices, etc. Unrecognized formats will be
+                                passed-through through as-is.
+
+  -h                            Display short usage and exit.
+  -H, --help                    Display full help and exit.
+  -V, --version                 Display the program version and exit.
+
++ zstd -H
+...
+Advanced options:
+...
++ zstd --help
+...
+Advanced options:
+...

externals/zstd/tests/cli-tests/basic/memlimit.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+echo "some data" > file
+
+println "+ zstd --memory=32LB file"
+zstd --memory=32LB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32LiB file"
+zstd --memory=32LiB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32A file"
+zstd --memory=32A file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32r82347dn83 file"
+zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32asbdf file"
+zstd --memory=32asbdf file && die "Should not allow bogus suffix"
+println "+ zstd --memory=hello file"
+zstd --memory=hello file && die "Should not allow non-numeric parameter"
+println "+ zstd --memory=1 file"
+zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
+rm file.zst
+println "+ zstd --memory=1K file"
+zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KB file"
+zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KiB file"
+zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1M file"
+zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MB file"
+zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MiB file"
+zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+
+rm file
+exit 0

externals/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact

@@ -0,0 +1,13 @@
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+Should allow numeric parameter without suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix

externals/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact

@@ -0,0 +1,13 @@
++ zstd --memory=32LB file
++ zstd --memory=32LiB file
++ zstd --memory=32A file
++ zstd --memory=32r82347dn83 file
++ zstd --memory=32asbdf file
++ zstd --memory=hello file
++ zstd --memory=1 file
++ zstd --memory=1K file
++ zstd --memory=1KB file
++ zstd --memory=1KiB file
++ zstd --memory=1M file
++ zstd --memory=1MB file
++ zstd --memory=1MiB file

externals/zstd/tests/cli-tests/basic/output_dir.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+println "+ zstd -r * --output-dir-mirror=\"\""
+zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!"
+println "+ zstd -r * --output-dir-flat=\"\""
+zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!"
+exit 0

externals/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact

@@ -0,0 +1,2 @@
+error: output dir cannot be empty string (did you mean to pass '.' instead?)
+error: output dir cannot be empty string (did you mean to pass '.' instead?)

externals/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact

@@ -0,0 +1,2 @@
++ zstd -r * --output-dir-mirror=""
++ zstd -r * --output-dir-flat=""

externals/zstd/tests/cli-tests/basic/version.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+zstd -V
+zstd --version

externals/zstd/tests/cli-tests/basic/version.sh.stdout.glob

@@ -0,0 +1,2 @@
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***

externals/zstd/tests/cli-tests/bin/cmp_size

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -e
+
+usage()
+{
+	printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n"
+}
+
+help()
+{
+	printf "Small utility to compare file sizes without printing them with set -x.\n\n"
+	usage
+}
+
+case "$1" in
+	-h) help; exit 0  ;;
+	--help) help; exit 0 ;;
+esac
+
+if ! test -f $2; then
+	printf "FILE1='%b' is not a file\n\n" "$2"
+	usage
+	exit 1
+fi
+
+if ! test -f $3; then
+	printf "FILE2='%b' is not a file\n\n" "$3"
+	usage
+	exit 1
+fi
+
+
+size1=$(wc -c < $2)
+size2=$(wc -c < $3)
+
+case "$1" in
+	-eq) [ "$size1" -eq "$size2" ] ;;
+	-ne) [ "$size1" -ne "$size2" ] ;;
+	-lt) [ "$size1" -lt "$size2" ] ;;
+	-le) [ "$size1" -le "$size2" ] ;;
+	-gt) [ "$size1" -gt "$size2" ] ;;
+	-ge) [ "$size1" -ge "$size2" ] ;;
+esac

externals/zstd/tests/cli-tests/bin/datagen

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"$DATAGEN_BIN" $@

externals/zstd/tests/cli-tests/bin/die

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+println "${*}" 1>&2
+exit 1

externals/zstd/tests/cli-tests/bin/println

@@ -0,0 +1,2 @@
+#!/bin/sh
+printf '%b\n' "${*}"

externals/zstd/tests/cli-tests/bin/unzstd

@@ -0,0 +1 @@
+zstd

externals/zstd/tests/cli-tests/bin/zstd

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+zstdname=$(basename $0)
+
+if [ -z "$EXEC_PREFIX" ]; then
+    "$ZSTD_SYMLINK_DIR/$zstdname" $@
+else
+    $EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@
+fi

externals/zstd/tests/cli-tests/bin/zstdcat

@@ -0,0 +1 @@
+zstd

externals/zstd/tests/cli-tests/bin/zstdgrep

@@ -0,0 +1,2 @@
+#!/bin/sh
+"$ZSTDGREP_BIN" $@

externals/zstd/tests/cli-tests/bin/zstdless

@@ -0,0 +1,2 @@
+#!/bin/sh
+"$ZSTDLESS_BIN" $@

externals/zstd/tests/cli-tests/cltools/setup

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+echo "1234" > file
+zstd file

externals/zstd/tests/cli-tests/cltools/zstdgrep.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+println "+ good path"
+zstdgrep "1234" file file.zst
+println "+ bad path"
+zstdgrep "1234" bad.zst

externals/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit

@@ -0,0 +1 @@
+1

externals/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact

@@ -0,0 +1 @@
+zstd: can't stat bad.zst : No such file or directory -- ignored 

externals/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob

@@ -0,0 +1,4 @@
++ good path
+file:1234
+file.zst:1234
++ bad path

externals/zstd/tests/cli-tests/cltools/zstdless.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+println "+ good path"
+zstdless file.zst
+println "+ pass parameters"
+zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd
+println "+ bad path"
+zstdless bad.zst >&2

externals/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact

@@ -0,0 +1,2 @@
+zstd: can't stat bad.zst : No such file or directory -- ignored 
+bad.zst: No such file or directory

externals/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob

@@ -0,0 +1,5 @@
++ good path
+1234
++ pass parameters
+1234
++ bad path

externals/zstd/tests/cli-tests/common/format.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+zstd_supports_format()
+{
+	zstd -h | grep > $INTOVOID -- "--format=$1"
+}
+
+format_extension()
+{
+	if [ "$1" = "zstd" ]; then
+		printf "zst"
+	elif [ "$1" = "gzip" ]; then
+		printf "gz"
+	else
+		printf "$1"
+	fi
+}

externals/zstd/tests/cli-tests/common/mtime.sh

@@ -0,0 +1,13 @@
+. "$COMMON/platform.sh"
+
+MTIME="stat -c %Y"
+case "$UNAME" in
+    Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
+esac
+
+assertSameMTime() {
+    MT1=$($MTIME "$1")
+    MT2=$($MTIME "$2")
+    echo MTIME $MT1 $MT2
+    [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
+}

externals/zstd/tests/cli-tests/common/permissions.sh

@@ -0,0 +1,18 @@
+. "$COMMON/platform.sh"
+
+GET_PERMS="stat -c %a"
+case "$UNAME" in
+    Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
+esac
+
+assertFilePermissions() {
+    STAT1=$($GET_PERMS "$1")
+    STAT2=$2
+    [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)"
+}
+
+assertSamePermissions() {
+    STAT1=$($GET_PERMS "$1")
+    STAT2=$($GET_PERMS "$2")
+    [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)"
+}

externals/zstd/tests/cli-tests/common/platform.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+UNAME=$(uname)
+
+isWindows=false
+INTOVOID="/dev/null"
+case "$UNAME" in
+  GNU) DEVDEVICE="/dev/random" ;;
+  *) DEVDEVICE="/dev/zero" ;;
+esac
+case "$OS" in
+  Windows*)
+    isWindows=true
+    INTOVOID="NUL"
+    DEVDEVICE="NUL"
+    ;;
+esac
+
+case "$UNAME" in
+  Darwin) MD5SUM="md5 -r" ;;
+  FreeBSD) MD5SUM="gmd5sum" ;;
+  NetBSD) MD5SUM="md5 -n" ;;
+  OpenBSD) MD5SUM="md5" ;;
+  *) MD5SUM="md5sum" ;;
+esac
+
+DIFF="diff"
+case "$UNAME" in
+  SunOS) DIFF="gdiff" ;;
+esac
+
+if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
+then
+    hasMT=""
+else
+    hasMT="true"
+fi

externals/zstd/tests/cli-tests/compression/adapt.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+# Test --adapt
+zstd -f file --adapt -c | zstd -t
+
+datagen -g100M > file100M
+
+# Pick parameters to force fast adaptation, even on slow systems
+zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
+
+# Adaption still happens with --no-progress
+zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"

externals/zstd/tests/cli-tests/compression/basic.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
+
+# Test compression flags and check that they work
+zstd file                 ; zstd -t file.zst
+zstd -f file              ; zstd -t file.zst
+zstd -f -z file           ; zstd -t file.zst
+zstd -f -k file           ; zstd -t file.zst
+zstd -f -C file           ; zstd -t file.zst
+zstd -f --check file      ; zstd -t file.zst
+zstd -f --no-check file   ; zstd -t file.zst
+zstd -f -- file           ; zstd -t file.zst
+
+# Test output file compression
+zstd -o file-out.zst ; zstd -t file-out.zst
+zstd -fo file-out.zst; zstd -t file-out.zst
+
+# Test compression to stdout
+zstd -c file       | zstd -t
+zstd --stdout file | zstd -t
+println bob | zstd | zstd -t
+
+# Test keeping input file when compressing to stdout in gzip mode
+if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
+    $ZSTD_SYMLINK_DIR/gzip -c file       | zstd -t ; test -f file
+    $ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file
+fi
+
+# Test --rm
+cp file file-rm
+zstd --rm file-rm; zstd -t file-rm.zst
+test ! -f file-rm

externals/zstd/tests/cli-tests/compression/compress-literals.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+# Test --[no-]compress-literals
+zstd file --no-compress-literals -1 -c       | zstd -t
+zstd file --no-compress-literals -19 -c      | zstd -t
+zstd file --no-compress-literals --fast=1 -c | zstd -t
+zstd file --compress-literals -1 -c          | zstd -t
+zstd file --compress-literals --fast=1 -c    | zstd -t

externals/zstd/tests/cli-tests/compression/format.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "$COMMON/format.sh"
+
+set -e
+
+# Test --format
+zstd --format=zstd file -f
+zstd -t file.zst
+for format in "gzip" "lz4" "xz" "lzma"; do
+	if zstd_supports_format $format; then
+		zstd --format=$format file
+		zstd -t file.$(format_extension $format)
+		zstd -c --format=$format file | zstd -t --format=$format
+	fi
+done

externals/zstd/tests/cli-tests/compression/golden.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+cp -r "$GOLDEN_DIR" golden/
+
+zstd -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/
+
+zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/
+
+# PR #3517 block splitter corruption test
+zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/

externals/zstd/tests/cli-tests/compression/gzip-compat.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
+
+# Test gzip specific compression option
+if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
+    $ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+    $ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+
+    # Test -n / --no-name: do not embed original filename in archive
+    $ZSTD_SYMLINK_DIR/gzip -n file           ; grep -qv file file.gz  ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+    $ZSTD_SYMLINK_DIR/gzip --no-name file    ; grep -qv file file.gz  ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+    $ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file
+fi

externals/zstd/tests/cli-tests/compression/levels.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+set -v
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f -q
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst -q
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+
+zstd -5000000000 -f file       && die "Level too large, must fail" ||:
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q    ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
+
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"

externals/zstd/tests/cli-tests/compression/levels.sh.stderr.exact

@@ -0,0 +1,69 @@
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+4 files decompressed : 262148 bytes total 
+
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f -q
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst -q
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+/*stdin*\           : 65537 bytes 
+
+zstd -5000000000 -f file       && die "Level too large, must fail" ||:
+error: numeric value overflows 32-bit unsigned int 
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
+error: numeric value overflows 32-bit unsigned int 
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q      ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q     ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q    ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
+
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"

externals/zstd/tests/cli-tests/compression/long-distance-matcher.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Test --long
+zstd -f file --long   ; zstd -t file.zst
+zstd -f file --long=20; zstd -t file.zst

externals/zstd/tests/cli-tests/compression/multi-threaded.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Test multi-threaded flags
+zstd --single-thread file -f -q         ; zstd -t file.zst
+zstd -T2 -f file -q                     ; zstd -t file.zst
+zstd --rsyncable -f file -q             ; zstd -t file.zst
+zstd -T0 -f file -q                     ; zstd -t file.zst
+zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
+zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
+
+# multi-thread decompression warning test
+zstd -T0 -f file -q                     ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
+zstd -T0 -f file -q                     ; zstd -t file.zst; zstd -T2 -d file.zst -o file4

externals/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact

@@ -0,0 +1,11 @@
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+file.zst            : 65537 bytes 
+Warning : decompression does not support multi-threading
+file.zst            : 65537 bytes 

externals/zstd/tests/cli-tests/compression/multiple-files.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -e
+
+# setup
+echo "file1" > file1
+echo "file2" > file2
+
+echo "Test zstd ./file1 - file2"
+rm -f ./file*.zst
+echo "stdin" | zstd ./file1 - ./file2 | zstd -d
+cat file1.zst | zstd -d
+cat file2.zst | zstd -d
+
+echo "Test zstd -d ./file1.zst - file2.zst"
+rm ./file1 ./file2
+echo "stdin" | zstd - | zstd -d ./file1.zst - file2.zst
+cat file1
+cat file2
+
+echo "zstd -d ./file1.zst - file2.zst -c"
+echo "stdin" | zstd | zstd -d ./file1.zst - file2.zst -c

externals/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact

@@ -0,0 +1,12 @@
+Test zstd ./file1 - file2
+stdin
+file1
+file2
+Test zstd -d ./file1.zst - file2.zst
+stdin
+file1
+file2
+zstd -d ./file1.zst - file2.zst -c
+file1
+stdin
+file2

externals/zstd/tests/cli-tests/compression/row-match-finder.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Test --[no-]row-match-finder
+zstd file -7f --row-match-finder
+zstd file -7f --no-row-match-finder

externals/zstd/tests/cli-tests/compression/setup

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+datagen > file0
+datagen > file1

externals/zstd/tests/cli-tests/compression/stream-size.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Test stream size & hint
+datagen -g7654 | zstd --stream-size=7654 | zstd -t
+datagen -g7654 | zstd --size-hint=7000   | zstd -t

externals/zstd/tests/cli-tests/compression/verbose-wlog.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+zstd < file -vv -19 -o file.19.zst
+zstd -vv -l file.19.zst
+
+zstd < file -vv -19 --long -o file.19.long.zst
+zstd -vv -l file.19.long.zst

externals/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob

@@ -0,0 +1,5 @@
+...
+*wlog=23*
+...
+*wlog=27*
+...

externals/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob

@@ -0,0 +1,5 @@
+...
+*Window Size: 8388608 B*
+...
+*Window Size: 134217728 B*
+...

externals/zstd/tests/cli-tests/compression/window-resize.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=30 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst

externals/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob

@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...

externals/zstd/tests/cli-tests/decompression/golden.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/"
+
+zstd -r -t "$GOLDEN_DIR"

externals/zstd/tests/cli-tests/decompression/pass-through.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+echo "" > 1
+echo "2" > 2
+echo "23" > 3
+echo "234" > 4
+echo "some data" > file
+
+println "+ passthrough enabled"
+
+zstd file
+
+# Test short files
+zstd -dc --pass-through 1 2 3 4
+
+# Test *cat symlinks
+zstdcat file
+"$ZSTD_SYMLINK_DIR/zcat" file
+"$ZSTD_SYMLINK_DIR/gzcat" file
+
+# Test multiple files with mix of compressed & not
+zstdcat file file.zst
+zstdcat file.zst file
+
+# Test --pass-through
+zstd -dc --pass-through file
+zstd -d --pass-through file -o pass-through-file
+
+# Test legacy implicit passthrough with -fc
+zstd -dcf file
+zstd -dcf file file.zst
+zstd -df < file
+zstd -dcf < file file.zst -
+zstd -dcf < file.zst file -
+
+$DIFF file pass-through-file
+
+println "+ passthrough disabled"
+
+# Test *cat
+zstdcat --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
+# Test zstd without implicit passthrough
+zstd -d file -o no-pass-through-file && die "should fail"
+zstd -d < file && die "should fail"
+
+# Test legacy implicit passthrough with -fc
+zstd --no-pass-through -dcf file && die "should fail"
+zstd --no-pass-through -dcf file file.zst && die "should fail"
+zstd --no-pass-through -df < file && die "should fail"
+zstd --no-pass-through -dcf < file file.zst - && die "should fail"
+zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:

externals/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact

@@ -0,0 +1,11 @@
+file                 :230.00%   (    10 B =>     23 B, file.zst) 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: file: unsupported format 

externals/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact

@@ -0,0 +1,25 @@
++ passthrough enabled
+
+2
+23
+234
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
++ passthrough disabled
+some data
+some data
+some data

externals/zstd/tests/cli-tests/dict-builder/empty-input.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+for i in $(seq 50); do
+    datagen -s$i > file$i
+done
+touch empty
+
+set -v
+zstd -q --train empty file*

externals/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact

@@ -0,0 +1 @@
+zstd -q --train empty file*

externals/zstd/tests/cli-tests/dict-builder/no-inputs.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+set -v
+zstd --train

externals/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit

@@ -0,0 +1 @@
+14

externals/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact

@@ -0,0 +1,5 @@
+zstd --train
+!  Warning : nb of samples too low for proper processing ! 
+!  Please provide _one file per sample_. 
+!  Alternatively, split files into fixed-size blocks representative of samples, with -B# 
+Error 14 : nb of samples too low

externals/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+if [ false ]; then
+	for seed in $(seq 100); do
+		datagen -g1000 -s$seed > file$seed
+	done
+
+	zstd --train -r . -o dict0 -qq
+
+	for seed in $(seq 101 200); do
+		datagen -g1000 -s$seed > file$seed
+	done
+
+	zstd --train -r . -o dict1 -qq
+
+	[ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match"
+
+	datagen -g1000 -s0 > file0
+fi
+
+set -v
+zstd files/0 -D dicts/0 -q
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
+zstd -t files/0.zst            && die "Must fail" ||:

externals/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact

@@ -0,0 +1,7 @@
+zstd files/0 -D dicts/0 -q
+zstd -t files/0.zst -D dicts/0
+files/0.zst         : 1000 bytes 
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
+files/0.zst : Decoding error (36) : Dictionary mismatch 
+zstd -t files/0.zst            && die "Must fail" ||:
+files/0.zst : Decoding error (36) : Dictionary mismatch 

externals/zstd/tests/cli-tests/dictionaries/golden.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/"
+
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst

externals/zstd/tests/cli-tests/dictionaries/setup

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+cp -r ../files .
+cp -r ../dicts .

externals/zstd/tests/cli-tests/dictionaries/setup_once

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+
+mkdir files/ dicts/
+
+for seed in $(seq 50); do
+	datagen -g1000 -s$seed > files/$seed
+done
+
+zstd --train -r files -o dicts/0 -qq
+
+for seed in $(seq 51 100); do
+	datagen -g1000 -s$seed > files/$seed
+done
+
+zstd --train -r files -o dicts/1 -qq
+
+cmp dicts/0 dicts/1 && die "dictionaries must not match!"
+
+datagen -g1000 > files/0

externals/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# motivated by issue #3523
+
+datagen > file
+mkdir out
+chmod 000 out
+
+zstd file -q --trace-file-stat -o out/file.zst
+zstd -tq out/file.zst
+
+chmod 777 out

externals/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact

@@ -0,0 +1,26 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, out/file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(-1, out/file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(out/file.zst)
+Trace:FileStat:  > UTIL_stat(-1, out/file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+zstd: out/file.zst: Permission denied
+zstd: can't stat out/file.zst : Permission denied -- ignored 

externals/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+chmod 642 file
+
+zstd file -q --trace-file-stat -o file.zst
+zstd -tq file.zst

externals/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact

@@ -0,0 +1,42 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_setFileStat(4, file.zst)
+Trace:FileStat:  > UTIL_stat(4, file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_chmod(file.zst, 0642)
+Trace:FileStat:   > fchmod
+Trace:FileStat:   < 0
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_utime(file.zst)
+Trace:FileStat: < 0

externals/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -cq --trace-file-stat > file.zst
+zstd -tq file.zst

externals/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact

@@ -0,0 +1,24 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 65537

externals/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -q --trace-file-stat -o file.zst
+zstd -tq file.zst

externals/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact

@@ -0,0 +1,24 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1

externals/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -cq --trace-file-stat > file.zst
+zstd -tq file.zst

externals/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact

@@ -0,0 +1,18 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < -1

externals/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+chmod 642 file.zst
+
+zstd -dq --trace-file-stat file.zst

externals/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact

@@ -0,0 +1,38 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_isDirectoryStat()
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file.zst, file)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_setFileStat(4, file)
+Trace:FileStat:  > UTIL_stat(4, file)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_chmod(file, 0642)
+Trace:FileStat:   > fchmod
+Trace:FileStat:   < 0
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_utime(file)
+Trace:FileStat: < 0

externals/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat file.zst > file

externals/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact

@@ -0,0 +1,18 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat:  > UTIL_stat(-1, file.zst)
+Trace:FileStat:  < 1
+Trace:FileStat:  > UTIL_isDirectoryStat()
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0

externals/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst -o file

externals/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact

@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat:  > UTIL_stat(-1, file)
+Trace:FileStat:  < 1
+Trace:FileStat: < 1

externals/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst > file

externals/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact

@@ -0,0 +1,14 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat:  > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat:  < 0
+Trace:FileStat: < 0

externals/zstd/tests/cli-tests/progress/no-progress.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+println >&2 "Tests cases where progress information should not be printed"
+
+for args in \
+	"" \
+	"--fake-stderr-is-console -q" \
+	"--fake-stderr-is-console -qq --progress" \
+	"--no-progress --fake-stderr-is-console" \
+	"--no-progress --fake-stderr-is-console -v"
+do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done

externals/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob

@@ -0,0 +1,96 @@
+Tests cases where progress information should not be printed
+args =
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --fake-stderr-is-console -q
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -qq --progress
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --no-progress --fake-stderr-is-console
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --no-progress --fake-stderr-is-console -v
+compress file to file
+*Zstandard CLI*
+hello*hello.zst*
+compress pipe to pipe
+*Zstandard CLI*
+*stdin*stdout*
+compress pipe to file
+*Zstandard CLI*
+*stdin*hello.zst*
+compress file to pipe
+*Zstandard CLI*
+*hello*stdout*
+compress 2 files
+*Zstandard CLI*
+*hello*hello.zst*
+*world*world.zst*
+2 files compressed*
+decompress file to file
+*Zstandard CLI*
+hello.zst*
+decompress pipe to pipe
+*Zstandard CLI*
+*stdin*
+decompress pipe to file
+*Zstandard CLI*
+*stdin*
+decompress file to pipe
+*Zstandard CLI*
+hello.zst*
+decompress 2 files
+*Zstandard CLI*
+hello.zst*
+world.zst*
+2 files decompressed*

externals/zstd/tests/cli-tests/progress/progress.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+println >&2 "Tests cases where progress information should be printed"
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+for args in \
+	"--progress" \
+	"--fake-stderr-is-console" \
+	"--progress --fake-stderr-is-console -q"; do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done

externals/zstd/tests/cli-tests/progress/progress.sh.stderr.glob

@@ -0,0 +1,62 @@
+Tests cases where progress information should be printed
+args = --progress
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --fake-stderr-is-console
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --progress --fake-stderr-is-console -q
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*

externals/zstd/tests/cli-tests/run.py

@@ -0,0 +1,731 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ##########################################################################
+
+import argparse
+import contextlib
+import copy
+import fnmatch
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import typing
+
+
+ZSTD_SYMLINKS = [
+    "zstd",
+    "zstdmt",
+    "unzstd",
+    "zstdcat",
+    "zcat",
+    "gzip",
+    "gunzip",
+    "gzcat",
+    "lzma",
+    "unlzma",
+    "xz",
+    "unxz",
+    "lz4",
+    "unlz4",
+]
+
+
+EXCLUDED_DIRS = {
+    "bin",
+    "common",
+    "scratch",
+}
+
+
+EXCLUDED_BASENAMES = {
+    "setup",
+    "setup_once",
+    "teardown",
+    "teardown_once",
+    "README.md",
+    "run.py",
+    ".gitignore",
+}
+
+EXCLUDED_SUFFIXES = [
+    ".exact",
+    ".glob",
+    ".ignore",
+    ".exit",
+]
+
+
+def exclude_dir(dirname: str) -> bool:
+    """
+    Should files under the directory :dirname: be excluded from the test runner?
+    """
+    if dirname in EXCLUDED_DIRS:
+        return True
+    return False
+
+
+def exclude_file(filename: str) -> bool:
+    """Should the file :filename: be excluded from the test runner?"""
+    if filename in EXCLUDED_BASENAMES:
+        return True
+    for suffix in EXCLUDED_SUFFIXES:
+        if filename.endswith(suffix):
+            return True
+    return False
+
+def read_file(filename: str) -> bytes:
+    """Reads the file :filename: and returns the contents as bytes."""
+    with open(filename, "rb") as f:
+        return f.read()
+
+
+def diff(a: bytes, b: bytes) -> str:
+    """Returns a diff between two different byte-strings :a: and :b:."""
+    assert a != b
+    with tempfile.NamedTemporaryFile("wb") as fa:
+        fa.write(a)
+        fa.flush()
+        with tempfile.NamedTemporaryFile("wb") as fb:
+            fb.write(b)
+            fb.flush()
+
+            diff_bytes = subprocess.run(["diff", fa.name, fb.name], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout
+            return diff_bytes.decode("utf8")
+
+
+def pop_line(data: bytes) -> typing.Tuple[typing.Optional[bytes], bytes]:
+    """
+    Pop the first line from :data: and returns the first line and the remainder
+    of the data as a tuple. If :data: is empty, returns :(None, data):. Otherwise
+    the first line always ends in a :\n:, even if it is the last line and :data:
+    doesn't end in :\n:.
+    """
+    NEWLINE = b"\n"
+
+    if data == b'':
+        return (None, data)
+
+    parts = data.split(NEWLINE, maxsplit=1)
+    line = parts[0] + NEWLINE
+    if len(parts) == 1:
+        return line, b''
+
+    return line, parts[1]
+
+
+def glob_line_matches(actual: bytes, expect: bytes) -> bool:
+    """
+    Does the `actual` line match the expected glob line `expect`?
+    """
+    return fnmatch.fnmatchcase(actual.strip(), expect.strip())
+
+
+def glob_diff(actual: bytes, expect: bytes) -> bytes:
+    """
+    Returns None if the :actual: content matches the expected glob :expect:,
+    otherwise returns the diff bytes.
+    """
+    diff = b''
+    actual_line, actual = pop_line(actual)
+    expect_line, expect = pop_line(expect)
+    while True:
+        # Handle end of file conditions - allow extra newlines
+        while expect_line is None and actual_line == b"\n":
+            actual_line, actual = pop_line(actual)
+        while actual_line is None and expect_line == b"\n":
+            expect_line, expect = pop_line(expect)
+
+        if expect_line is None and actual_line is None:
+            if diff == b'':
+                return None
+            return diff
+        elif expect_line is None:
+            diff += b"---\n"
+            while actual_line != None:
+                diff += b"> "
+                diff += actual_line
+                actual_line, actual = pop_line(actual)
+            return diff
+        elif actual_line is None:
+            diff += b"---\n"
+            while expect_line != None:
+                diff += b"< "
+                diff += expect_line
+                expect_line, expect = pop_line(expect)
+            return diff
+
+        assert expect_line is not None
+        assert actual_line is not None
+
+        if expect_line == b'...\n':
+            next_expect_line, expect = pop_line(expect)
+            if next_expect_line is None:
+                if diff == b'':
+                    return None
+                return diff
+            while not glob_line_matches(actual_line, next_expect_line):
+                actual_line, actual = pop_line(actual)
+                if actual_line is None:
+                    diff += b"---\n"
+                    diff += b"< "
+                    diff += next_expect_line
+                    return diff
+            expect_line = next_expect_line
+            continue
+
+        if not glob_line_matches(actual_line, expect_line):
+            diff += b'---\n'
+            diff += b'< ' + expect_line
+            diff += b'> ' + actual_line
+
+        actual_line, actual = pop_line(actual)
+        expect_line, expect = pop_line(expect)
+
+
+class Options:
+    """Options configuring how to run a :TestCase:."""
+    def __init__(
+        self,
+        env: typing.Dict[str, str],
+        timeout: typing.Optional[int],
+        verbose: bool,
+        preserve: bool,
+        scratch_dir: str,
+        test_dir: str,
+        set_exact_output: bool,
+    ) -> None:
+        self.env = env
+        self.timeout = timeout
+        self.verbose = verbose
+        self.preserve = preserve
+        self.scratch_dir = scratch_dir
+        self.test_dir = test_dir
+        self.set_exact_output = set_exact_output
+
+
+class TestCase:
+    """
+    Logic and state related to running a single test case.
+
+    1. Initialize the test case.
+    2. Launch the test case with :TestCase.launch():.
+       This will start the test execution in a subprocess, but
+       not wait for completion. So you could launch multiple test
+       cases in parallel. This will now print any test output.
+    3. Analyze the results with :TestCase.analyze():. This will
+       join the test subprocess, check the results against the
+       expectations, and print the results to stdout.
+
+    :TestCase.run(): is also provided which combines the launch & analyze
+    steps for single-threaded use-cases.
+
+    All other methods, prefixed with _, are private helper functions.
+    """
+    def __init__(self, test_filename: str, options: Options) -> None:
+        """
+        Initialize the :TestCase: for the test located in :test_filename:
+        with the given :options:.
+        """
+        self._opts = options
+        self._test_file = test_filename
+        self._test_name = os.path.normpath(
+            os.path.relpath(test_filename, start=self._opts.test_dir)
+        )
+        self._success = {}
+        self._message = {}
+        self._test_stdin = None
+        self._scratch_dir = os.path.abspath(os.path.join(self._opts.scratch_dir, self._test_name))
+
+    @property
+    def name(self) -> str:
+        """Returns the unique name for the test."""
+        return self._test_name
+
+    def launch(self) -> None:
+        """
+        Launch the test case as a subprocess, but do not block on completion.
+        This allows users to run multiple tests in parallel. Results aren't yet
+        printed out.
+        """
+        self._launch_test()
+
+    def analyze(self) -> bool:
+        """
+        Must be called after :TestCase.launch():. Joins the test subprocess and
+        checks the results against expectations. Finally prints the results to
+        stdout and returns the success.
+        """
+        self._join_test()
+        self._check_exit()
+        self._check_stderr()
+        self._check_stdout()
+        self._analyze_results()
+        return self._succeeded
+
+    def run(self) -> bool:
+        """Shorthand for combining both :TestCase.launch(): and :TestCase.analyze():."""
+        self.launch()
+        return self.analyze()
+
+    def _log(self, *args, **kwargs) -> None:
+        """Logs test output."""
+        print(file=sys.stdout, *args, **kwargs)
+
+    def _vlog(self, *args, **kwargs) -> None:
+        """Logs verbose test output."""
+        if self._opts.verbose:
+            print(file=sys.stdout, *args, **kwargs)
+
+    def _test_environment(self) -> typing.Dict[str, str]:
+        """
+        Returns the environment to be used for the
+        test subprocess.
+        """
+        # We want to omit ZSTD cli flags so tests will be consistent across environments
+        env = {k: v for k, v in os.environ.items() if not k.startswith("ZSTD")}
+        for k, v in self._opts.env.items():
+            self._vlog(f"${k}='{v}'")
+            env[k] = v
+        return env
+
+    def _launch_test(self) -> None:
+        """Launch the test subprocess, but do not join it."""
+        args = [os.path.abspath(self._test_file)]
+        stdin_name = f"{self._test_file}.stdin"
+        if os.path.exists(stdin_name):
+            self._test_stdin = open(stdin_name, "rb")
+            stdin = self._test_stdin
+        else:
+            stdin = subprocess.DEVNULL
+        cwd = self._scratch_dir
+        env = self._test_environment()
+        self._test_process = subprocess.Popen(
+            args=args,
+            stdin=stdin,
+            cwd=cwd,
+            env=env,
+            stderr=subprocess.PIPE,
+            stdout=subprocess.PIPE
+        )
+
+    def _join_test(self) -> None:
+        """Join the test process and save stderr, stdout, and the exit code."""
+        (stdout, stderr) = self._test_process.communicate(timeout=self._opts.timeout)
+        self._output = {}
+        self._output["stdout"] = stdout
+        self._output["stderr"] = stderr
+        self._exit_code = self._test_process.returncode
+        self._test_process = None
+        if self._test_stdin is not None:
+            self._test_stdin.close()
+            self._test_stdin = None
+
+    def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None:
+        """
+        Check the output named :out_name: for an exact match against the :expected: content.
+        Saves the success and message.
+        """
+        check_name = f"check_{out_name}"
+        actual = self._output[out_name]
+        if actual == expected:
+            self._success[check_name] = True
+            self._message[check_name] = f"{out_name} matches!"
+        else:
+            self._success[check_name] = False
+            self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}"
+
+            if self._opts.set_exact_output:
+                with open(exact_name, "wb") as f:
+                    f.write(actual)
+
+    def _check_output_glob(self, out_name: str, expected: bytes) -> None:
+        """
+        Check the output named :out_name: for a glob match against the :expected: glob.
+        Saves the success and message.
+        """
+        check_name = f"check_{out_name}"
+        actual = self._output[out_name]
+        diff = glob_diff(actual, expected)
+        if diff is None:
+            self._success[check_name] = True
+            self._message[check_name] = f"{out_name} matches!"
+        else:
+            utf8_diff = diff.decode('utf8')
+            self._success[check_name] = False
+            self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{utf8_diff}"
+
+    def _check_output(self, out_name: str) -> None:
+        """
+        Checks the output named :out_name: for a match against the expectation.
+        We check for a .exact, .glob, and a .ignore file. If none are found we
+        expect that the output should be empty.
+
+        If :Options.preserve: was set then we save the scratch directory and
+        save the stderr, stdout, and exit code to the scratch directory for
+        debugging.
+        """
+        if self._opts.preserve:
+            # Save the output to the scratch directory
+            actual_name = os.path.join(self._scratch_dir, f"{out_name}")
+            with open(actual_name, "wb") as f:
+                    f.write(self._output[out_name])
+
+        exact_name = f"{self._test_file}.{out_name}.exact"
+        glob_name = f"{self._test_file}.{out_name}.glob"
+        ignore_name = f"{self._test_file}.{out_name}.ignore"
+
+        if os.path.exists(exact_name):
+            return self._check_output_exact(out_name, read_file(exact_name), exact_name)
+        elif os.path.exists(glob_name):
+            return self._check_output_glob(out_name, read_file(glob_name))
+        else:
+            check_name = f"check_{out_name}"
+            self._success[check_name] = True
+            self._message[check_name] = f"{out_name} ignored!"
+
+    def _check_stderr(self) -> None:
+        """Checks the stderr output against the expectation."""
+        self._check_output("stderr")
+
+    def _check_stdout(self) -> None:
+        """Checks the stdout output against the expectation."""
+        self._check_output("stdout")
+
+    def _check_exit(self) -> None:
+        """
+        Checks the exit code against expectations. If a .exit file
+        exists, we expect that the exit code matches the contents.
+        Otherwise we expect the exit code to be zero.
+
+        If :Options.preserve: is set we save the exit code to the
+        scratch directory under the filename "exit".
+        """
+        if self._opts.preserve:
+            exit_name = os.path.join(self._scratch_dir, "exit")
+            with open(exit_name, "w") as f:
+                f.write(str(self._exit_code) + "\n")
+        exit_name = f"{self._test_file}.exit"
+        if os.path.exists(exit_name):
+            exit_code: int = int(read_file(exit_name))
+        else:
+            exit_code: int = 0
+        if exit_code == self._exit_code:
+            self._success["check_exit"] = True
+            self._message["check_exit"] = "Exit code matches!"
+        else:
+            self._success["check_exit"] = False
+            self._message["check_exit"] = f"Exit code mismatch! Expected {exit_code} but got {self._exit_code}"
+
+    def _analyze_results(self) -> None:
+        """
+        After all tests have been checked, collect all the successes
+        and messages, and print the results to stdout.
+        """
+        STATUS = {True: "PASS", False: "FAIL"}
+        checks = sorted(self._success.keys())
+        self._succeeded = all(self._success.values())
+        self._log(f"{STATUS[self._succeeded]}: {self._test_name}")
+
+        if not self._succeeded or self._opts.verbose:
+            for check in checks:
+                if self._opts.verbose or not self._success[check]:
+                    self._log(f"{STATUS[self._success[check]]}: {self._test_name}.{check}")
+                    self._log(self._message[check])
+
+        self._log("----------------------------------------")
+
+
+class TestSuite:
+    """
+    Setup & teardown test suite & cases.
+    This class is intended to be used as a context manager.
+
+    TODO: Make setup/teardown failure emit messages, not throw exceptions.
+    """
+    def __init__(self, test_directory: str, options: Options) -> None:
+        self._opts = options
+        self._test_dir = os.path.abspath(test_directory)
+        rel_test_dir = os.path.relpath(test_directory, start=self._opts.test_dir)
+        assert not rel_test_dir.startswith(os.path.sep)
+        self._scratch_dir = os.path.normpath(os.path.join(self._opts.scratch_dir, rel_test_dir))
+
+    def __enter__(self) -> 'TestSuite':
+        self._setup_once()
+        return self
+
+    def __exit__(self, _exc_type, _exc_value, _traceback) -> None:
+        self._teardown_once()
+
+    @contextlib.contextmanager
+    def test_case(self, test_basename: str) -> TestCase:
+        """
+        Context manager for a test case in the test suite.
+        Pass the basename of the test relative to the :test_directory:.
+        """
+        assert os.path.dirname(test_basename) == ""
+        try:
+            self._setup(test_basename)
+            test_filename = os.path.join(self._test_dir, test_basename)
+            yield TestCase(test_filename, self._opts)
+        finally:
+            self._teardown(test_basename)
+
+    def _remove_scratch_dir(self, dir: str) -> None:
+        """Helper to remove a scratch directory with sanity checks"""
+        assert "scratch" in dir
+        assert dir.startswith(self._scratch_dir)
+        assert os.path.exists(dir)
+        shutil.rmtree(dir)
+
+    def _setup_once(self) -> None:
+        if os.path.exists(self._scratch_dir):
+            self._remove_scratch_dir(self._scratch_dir)
+        os.makedirs(self._scratch_dir)
+        setup_script = os.path.join(self._test_dir, "setup_once")
+        if os.path.exists(setup_script):
+            self._run_script(setup_script, cwd=self._scratch_dir)
+
+    def _teardown_once(self) -> None:
+        assert os.path.exists(self._scratch_dir)
+        teardown_script = os.path.join(self._test_dir, "teardown_once")
+        if os.path.exists(teardown_script):
+            self._run_script(teardown_script, cwd=self._scratch_dir)
+        if not self._opts.preserve:
+            self._remove_scratch_dir(self._scratch_dir)
+
+    def _setup(self, test_basename: str) -> None:
+        test_scratch_dir = os.path.join(self._scratch_dir, test_basename)
+        assert not os.path.exists(test_scratch_dir)
+        os.makedirs(test_scratch_dir)
+        setup_script = os.path.join(self._test_dir, "setup")
+        if os.path.exists(setup_script):
+            self._run_script(setup_script, cwd=test_scratch_dir)
+
+    def _teardown(self, test_basename: str) -> None:
+        test_scratch_dir = os.path.join(self._scratch_dir, test_basename)
+        assert os.path.exists(test_scratch_dir)
+        teardown_script = os.path.join(self._test_dir, "teardown")
+        if os.path.exists(teardown_script):
+            self._run_script(teardown_script, cwd=test_scratch_dir)
+        if not self._opts.preserve:
+            self._remove_scratch_dir(test_scratch_dir)
+
+    def _run_script(self, script: str, cwd: str) -> None:
+        env = copy.copy(os.environ)
+        for k, v in self._opts.env.items():
+            env[k] = v
+        try:
+            subprocess.run(
+                args=[script],
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=cwd,
+                env=env,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"{script} failed with exit code {e.returncode}!")
+            print(f"stderr:\n{e.stderr}")
+            print(f"stdout:\n{e.stdout}")
+            raise
+
+TestSuites = typing.Dict[str, typing.List[str]]
+
+def get_all_tests(options: Options) -> TestSuites:
+    """
+    Find all the test in the test directory and return the test suites.
+    """
+    test_suites = {}
+    for root, dirs, files in os.walk(options.test_dir, topdown=True):
+        dirs[:] = [d for d in dirs if not exclude_dir(d)]
+        test_cases = []
+        for file in files:
+            if not exclude_file(file):
+                test_cases.append(file)
+        assert root == os.path.normpath(root)
+        test_suites[root] = test_cases
+    return test_suites
+
+
+def resolve_listed_tests(
+    tests: typing.List[str], options: Options
+) -> TestSuites:
+    """
+    Resolve the list of tests passed on the command line into their
+    respective test suites. Tests can either be paths, or test names
+    relative to the test directory.
+    """
+    test_suites = {}
+    for test in tests:
+        if not os.path.exists(test):
+            test = os.path.join(options.test_dir, test)
+            if not os.path.exists(test):
+                raise RuntimeError(f"Test {test} does not exist!")
+
+        test = os.path.normpath(os.path.abspath(test))
+        assert test.startswith(options.test_dir)
+        test_suite = os.path.dirname(test)
+        test_case = os.path.basename(test)
+        test_suites.setdefault(test_suite, []).append(test_case)
+
+    return test_suites
+
+def run_tests(test_suites: TestSuites, options: Options) -> bool:
+    """
+    Runs all the test in the :test_suites: with the given :options:.
+    Prints the results to stdout.
+    """
+    tests = {}
+    for test_dir, test_files in test_suites.items():
+        with TestSuite(test_dir, options) as test_suite:
+            test_files = sorted(set(test_files))
+            for test_file in test_files:
+                with test_suite.test_case(test_file) as test_case:
+                    tests[test_case.name] = test_case.run()
+
+    successes = 0
+    for test, status in tests.items():
+        if status:
+            successes += 1
+        else:
+            print(f"FAIL: {test}")
+    if successes == len(tests):
+        print(f"PASSED all {len(tests)} tests!")
+        return True
+    else:
+        print(f"FAILED {len(tests) - successes} / {len(tests)} tests!")
+        return False
+
+
+def setup_zstd_symlink_dir(zstd_symlink_dir: str, zstd: str) -> None:
+    assert os.path.join("bin", "symlinks") in zstd_symlink_dir
+    if not os.path.exists(zstd_symlink_dir):
+        os.makedirs(zstd_symlink_dir)
+    for symlink in ZSTD_SYMLINKS:
+        path = os.path.join(zstd_symlink_dir, symlink)
+        if os.path.exists(path):
+            os.remove(path)
+        os.symlink(zstd, path)
+
+if __name__ == "__main__":
+    CLI_TEST_DIR = os.path.dirname(sys.argv[0])
+    REPO_DIR = os.path.join(CLI_TEST_DIR, "..", "..")
+    PROGRAMS_DIR = os.path.join(REPO_DIR, "programs")
+    TESTS_DIR = os.path.join(REPO_DIR, "tests")
+    ZSTD_PATH = os.path.join(PROGRAMS_DIR, "zstd")
+    ZSTDGREP_PATH = os.path.join(PROGRAMS_DIR, "zstdgrep")
+    ZSTDLESS_PATH = os.path.join(PROGRAMS_DIR, "zstdless")
+    DATAGEN_PATH = os.path.join(TESTS_DIR, "datagen")
+
+    parser = argparse.ArgumentParser(
+        (
+            "Runs the zstd CLI tests. Exits nonzero on failure. Default arguments are\n"
+            "generally correct. Pass --preserve to preserve test output for debugging,\n"
+            "and --verbose to get verbose test output.\n"
+        )
+    )
+    parser.add_argument(
+        "--preserve",
+        action="store_true",
+        help="Preserve the scratch directory TEST_DIR/scratch/ for debugging purposes."
+    )
+    parser.add_argument("--verbose", action="store_true", help="Verbose test output.")
+    parser.add_argument("--timeout", default=200, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.")
+    parser.add_argument(
+        "--exec-prefix",
+        default=None,
+        help="Sets the EXEC_PREFIX environment variable. Prefix to invocations of the zstd CLI."
+    )
+    parser.add_argument(
+        "--zstd",
+        default=ZSTD_PATH,
+        help="Sets the ZSTD_BIN environment variable. Path of the zstd CLI."
+    )
+    parser.add_argument(
+        "--zstdgrep",
+        default=ZSTDGREP_PATH,
+        help="Sets the ZSTDGREP_BIN environment variable. Path of the zstdgrep CLI."
+    )
+    parser.add_argument(
+        "--zstdless",
+        default=ZSTDLESS_PATH,
+        help="Sets the ZSTDLESS_BIN environment variable. Path of the zstdless CLI."
+    )
+    parser.add_argument(
+        "--datagen",
+        default=DATAGEN_PATH,
+        help="Sets the DATAGEN_BIN environment variable. Path to the datagen CLI."
+    )
+    parser.add_argument(
+        "--test-dir",
+        default=CLI_TEST_DIR,
+        help=(
+            "Runs the tests under this directory. "
+            "Adds TEST_DIR/bin/ to path. "
+            "Scratch directory located in TEST_DIR/scratch/."
+        )
+    )
+    parser.add_argument(
+        "--set-exact-output",
+        action="store_true",
+        help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists"
+    )
+    parser.add_argument(
+        "tests",
+        nargs="*",
+        help="Run only these test cases. Can either be paths or test names relative to TEST_DIR/"
+    )
+    args = parser.parse_args()
+
+    if args.timeout <= 0:
+        args.timeout = None
+
+    args.test_dir = os.path.normpath(os.path.abspath(args.test_dir))
+    bin_dir = os.path.abspath(os.path.join(args.test_dir, "bin"))
+    zstd_symlink_dir = os.path.join(bin_dir, "symlinks")
+    scratch_dir = os.path.join(args.test_dir, "scratch")
+
+    setup_zstd_symlink_dir(zstd_symlink_dir, os.path.abspath(args.zstd))
+
+    env = {}
+    if args.exec_prefix is not None:
+        env["EXEC_PREFIX"] = args.exec_prefix
+    env["ZSTD_SYMLINK_DIR"] = zstd_symlink_dir
+    env["ZSTD_REPO_DIR"] = os.path.abspath(REPO_DIR)
+    env["DATAGEN_BIN"] = os.path.abspath(args.datagen)
+    env["ZSTDGREP_BIN"] = os.path.abspath(args.zstdgrep)
+    env["ZSTDLESS_BIN"] = os.path.abspath(args.zstdless)
+    env["COMMON"] = os.path.abspath(os.path.join(args.test_dir, "common"))
+    env["PATH"] = bin_dir + ":" + os.getenv("PATH", "")
+    env["LC_ALL"] = "C"
+
+    opts = Options(
+        env=env,
+        timeout=args.timeout,
+        verbose=args.verbose,
+        preserve=args.preserve,
+        test_dir=args.test_dir,
+        scratch_dir=scratch_dir,
+        set_exact_output=args.set_exact_output,
+    )
+
+    if len(args.tests) == 0:
+        tests = get_all_tests(opts)
+    else:
+        tests = resolve_listed_tests(args.tests, opts)
+
+    success = run_tests(tests, opts)
+    if success:
+        sys.exit(0)
+    else:
+        sys.exit(1)

externals/zstd/tests/cli-tests/zstd-symlinks/setup

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+println "hello" > hello
+println "world" > world
+zstd hello world