summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/zstd/tests/cli-tests
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/zstd/tests/cli-tests')
-rw-r--r--3rdparty/zstd/tests/cli-tests/.gitignore6
-rw-r--r--3rdparty/zstd/tests/cli-tests/README.md258
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/basic/help.sh10
-rw-r--r--3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob34
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/basic/memlimit.sh40
-rw-r--r--3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact13
-rw-r--r--3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact13
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/basic/output_dir.sh7
-rw-r--r--3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact2
-rw-r--r--3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact2
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/basic/version.sh6
-rw-r--r--3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob2
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/cmp_size44
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/datagen3
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/die4
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/println2
l---------3rdparty/zstd/tests/cli-tests/bin/unzstd1
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/zstd9
l---------3rdparty/zstd/tests/cli-tests/bin/zstdcat1
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/zstdgrep2
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/bin/zstdless2
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/cltools/setup6
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh8
-rw-r--r--3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit1
-rw-r--r--3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact1
-rw-r--r--3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob4
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh10
-rw-r--r--3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact2
-rw-r--r--3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob5
-rw-r--r--3rdparty/zstd/tests/cli-tests/common/format.sh19
-rw-r--r--3rdparty/zstd/tests/cli-tests/common/mtime.sh13
-rw-r--r--3rdparty/zstd/tests/cli-tests/common/permissions.sh18
-rw-r--r--3rdparty/zstd/tests/cli-tests/common/platform.sh37
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/adapt.sh14
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/basic.sh36
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh10
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/format.sh16
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/golden.sh16
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh17
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/levels.sh62
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact69
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh7
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh15
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact11
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh21
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact12
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh7
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/setup7
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/stream-size.sh7
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh11
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob5
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob5
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/compression/window-resize.sh9
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore0
-rw-r--r--3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob3
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/decompression/golden.sh7
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh57
-rw-r--r--3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact11
-rw-r--r--3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact25
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh9
-rw-r--r--3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact1
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh3
-rw-r--r--3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit1
-rw-r--r--3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact5
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh29
-rw-r--r--3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact7
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh9
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/dictionaries/setup6
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/dictionaries/setup_once24
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh12
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact26
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh9
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact42
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh8
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact24
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh8
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact24
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh8
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact18
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh8
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact38
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh7
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact18
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh7
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact20
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh7
-rw-r--r--3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact14
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/progress/no-progress.sh46
-rw-r--r--3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob96
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/progress/progress.sh41
-rw-r--r--3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob62
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/run.py731
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup6
-rwxr-xr-x3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh12
-rw-r--r--3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact8
95 files changed, 2439 insertions, 0 deletions
diff --git a/3rdparty/zstd/tests/cli-tests/.gitignore b/3rdparty/zstd/tests/cli-tests/.gitignore
new file mode 100644
index 00000000000..0ad01b24eff
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/.gitignore
@@ -0,0 +1,6 @@
+!bin/
+!datagen
+!zstdcat
+
+scratch/
+bin/symlinks
diff --git a/3rdparty/zstd/tests/cli-tests/README.md b/3rdparty/zstd/tests/cli-tests/README.md
new file mode 100644
index 00000000000..7ca07c3e9ac
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/README.md
@@ -0,0 +1,258 @@
+# CLI tests
+
+The CLI tests are focused on testing the zstd CLI.
+They are intended to be simple tests that the CLI and arguments work as advertised.
+They are not intended to test the library, only the code in `programs/`.
+The library will get incidental coverage, but if you find yourself trying to trigger a specific condition in the library, this is the wrong tool.
+
+## Test runner usage
+
+The test runner `run.py` will run tests against the in-tree build of `zstd` and `datagen` by default. Which means that `zstd` and `datagen` must be built.
+
+The `zstd` binary used can be passed with `--zstd /path/to/zstd`.
+Additionally, to run `zstd` through a tool like `valgrind` or `qemu`, set the `--exec-prefix 'valgrind -q'` flag.
+
+Similarly, the `--datagen`, and `--zstdgrep` flags can be set to specify
+the paths to their respective binaries. However, these tools do not use
+the `EXEC_PREFIX`.
+
+Each test executes in its own scratch directory under `scratch/test/name`. E.g. `scratch/basic/help.sh/`. Normally these directories are removed after the test executes. However, the `--preserve` flag will preserve these directories after execution, and save the tests exit code, stdout, and stderr in the scratch directory to `exit`, `stderr`, and `stdout` respectively. This can be useful for debugging/editing a test and updating the expected output.
+
+### Running all the tests
+
+By default the test runner `run.py` will run all the tests, and report the results.
+
+Examples:
+
+```
+./run.py
+./run.py --preserve
+./run.py --zstd ../../build/programs/zstd --datagen ../../build/tests/datagen
+```
+
+### Running specific tests
+
+A set of test names can be passed to the test runner `run.py` to only execute those tests.
+This can be useful for writing or debugging a test, especially with `--preserve`.
+
+The test name can either be the path to the test file, or the test name, which is the path relative to the test directory.
+
+Examples:
+
+```
+./run.py basic/help.sh
+./run.py --preserve basic/help.sh basic/version.sh
+./run.py --preserve --verbose basic/help.sh
+```
+
+### Updating exact output
+
+If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
+
+Example:
+```
+./run.py --set-exact-output
+./run.py basic/help.sh --set-exact-output
+```
+
+## Writing a test
+
+Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
+After the script executes, the exit code, stderr, and stdout are compared against the expectations.
+
+Each test is run in a clean directory that the test can use for intermediate files. This directory will be cleaned up at the end of the test, unless `--preserve` is passed to the test runner. Additionally, the `setup` script can prepare the directory before the test runs.
+
+### Calling zstd, utilities, and environment variables
+
+The `$PATH` for tests is prepended with the `bin/` sub-directory, which contains helper scripts for ease of testing.
+The `zstd` binary will call the zstd binary specified by `run.py` with the correct `$EXEC_PREFIX`.
+Similarly, `datagen`, `unzstd`, `zstdgrep`, `zstdcat`, etc, are provided.
+
+Helper utilities like `cmp_size`, `println`, and `die` are provided here too. See their scripts for details.
+
+Common shell script libraries are provided under `common/`, with helper variables and functions. They can be sourced with `source "$COMMON/library.sh`.
+
+Lastly, environment variables are provided for testing, which can be listed when calling `run.py` with `--verbose`.
+They are generally used by the helper scripts in `bin/` to coordinate everything.
+
+### Basic test case
+
+When executing your `$TEST` executable, by default the exit code is expected to be `0`. However, you can provide an alternate expected exit code in a `$TEST.exit` file.
+
+When executing your `$TEST` executable, by default the expected stderr and stdout are empty. However, you can override the default by providing one of three files:
+
+* `$TEST.{stdout,stderr}.exact`
+* `$TEST.{stdout,stderr}.glob`
+* `$TEST.{stdout,stderr}.ignore`
+
+If you provide a `.exact` file, the output is expected to exactly match, byte-for-byte.
+
+If you provide a `.glob` file, the output is expected to match the expected file, where each line is interpreted as a glob syntax. Additionally, a line containing only `...` matches all lines until the next expected line matches.
+
+If you provide a `.ignore` file, the output is ignored.
+
+#### Passing examples
+
+All these examples pass.
+
+Exit 1, and change the expectation to be 1.
+
+```
+exit-1.sh
+---
+#!/bin/sh
+exit 1
+---
+
+exit-1.sh.exit
+---
+1
+---
+```
+
+Check the stdout output exactly matches.
+
+```
+echo.sh
+---
+#!/bin/sh
+echo "hello world"
+---
+
+echo.sh.stdout.exact
+---
+hello world
+---
+```
+
+Check the stderr output using a glob.
+
+```
+random.sh
+---
+#!/bin/sh
+head -c 10 < /dev/urandom | xxd >&2
+---
+
+random.sh.stderr.glob
+---
+00000000: * * * * * *
+```
+
+Multiple lines can be matched with ...
+
+```
+random-num-lines.sh
+---
+#!/bin/sh
+echo hello
+seq 0 $RANDOM
+echo world
+---
+
+random-num-lines.sh.stdout.glob
+---
+hello
+0
+...
+world
+---
+```
+
+#### Failing examples
+
+Exit code is expected to be 0, but is 1.
+
+```
+exit-1.sh
+---
+#!/bin/sh
+exit 1
+---
+```
+
+Stdout is expected to be empty, but isn't.
+
+```
+echo.sh
+---
+#!/bin/sh
+echo hello world
+```
+
+Stderr is expected to be hello but is world.
+
+```
+hello.sh
+---
+#!/bin/sh
+echo world >&2
+---
+
+hello.sh.stderr.exact
+---
+hello
+---
+```
+
+### Setup & teardown scripts
+
+Finally, test writing can be eased with setup and teardown scripts.
+Each directory in the test directory is a test-suite consisting of all tests within that directory (but not sub-directories).
+This test suite can come with 4 scripts to help test writing:
+
+* `setup_once`
+* `teardown_once`
+* `setup`
+* `teardown`
+
+The `setup_once` and `teardown_once` are run once before and after all the tests in the suite respectively.
+They operate in the scratch directory for the test suite, which is the parent directory of each scratch directory for each test case.
+They can do work that is shared between tests to improve test efficiency.
+For example, the `dictionaries/setup_once` script builds several dictionaries, for use in the `dictionaries` tests.
+
+The `setup` and `teardown` scripts run before and after each test case respectively, in the test case's scratch directory.
+These scripts can do work that is shared between test cases to make tests more succinct.
+For example, the `dictionaries/setup` script copies the dictionaries built by the `dictionaries/setup_once` script into the test's scratch directory, to make them easier to use, and make sure they aren't accidentally modified.
+
+#### Examples
+
+```
+basic/setup
+---
+#!/bin/sh
+# Create some files for testing with
+datagen > file
+datagen > file0
+datagen > file1
+---
+
+basic/test.sh
+---
+#!/bin/sh
+zstd file file0 file1
+---
+
+dictionaries/setup_once
+---
+#!/bin/sh
+set -e
+
+mkdir files/ dicts/
+for i in $(seq 10); do
+ datagen -g1000 > files/$i
+done
+
+zstd --train -r files/ -o dicts/0
+---
+
+dictionaries/setup
+---
+#!/bin/sh
+
+# Runs in the test case's scratch directory.
+# The test suite's scratch directory that
+# `setup_once` operates in is the parent directory.
+cp -r ../files ../dicts .
+---
+```
diff --git a/3rdparty/zstd/tests/cli-tests/basic/help.sh b/3rdparty/zstd/tests/cli-tests/basic/help.sh
new file mode 100755
index 00000000000..927c3ffbcc5
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/help.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+println "+ zstd -h"
+zstd -h
+println "+ zstd -H"
+zstd -H
+println "+ zstd --help"
+zstd --help
diff --git a/3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob
new file mode 100644
index 00000000000..21bc28c3fa8
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob
@@ -0,0 +1,34 @@
++ zstd -h
+Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
+
+Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
+
+Options:
+ -o OUTPUT Write output to a single file, OUTPUT.
+ -k, --keep Preserve INPUT file(s). *Default*
+ --rm Remove INPUT file(s) after successful (de)compression.
+
+ -# Desired compression level, where `#` is a number between 1 and 19;
+ lower numbers provide faster compression, higher numbers yield
+ better compression ratios. *Default: 3*
+
+ -d, --decompress Perform decompression.
+ -D DICT Use DICT as the dictionary for compression or decompression.
+
+ -f, --force Disable input and output checks. Allows overwriting existing files,
+ receiving input from the console, printing output to STDOUT, and
+ operating on links, block devices, etc. Unrecognized formats will be
+ passed-through through as-is.
+
+ -h Display short usage and exit.
+ -H, --help Display full help and exit.
+ -V, --version Display the program version and exit.
+
++ zstd -H
+...
+Advanced options:
+...
++ zstd --help
+...
+Advanced options:
+...
diff --git a/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh
new file mode 100755
index 00000000000..88d734d0d9d
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+echo "some data" > file
+
+println "+ zstd --memory=32LB file"
+zstd --memory=32LB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32LiB file"
+zstd --memory=32LiB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32A file"
+zstd --memory=32A file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32r82347dn83 file"
+zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32asbdf file"
+zstd --memory=32asbdf file && die "Should not allow bogus suffix"
+println "+ zstd --memory=hello file"
+zstd --memory=hello file && die "Should not allow non-numeric parameter"
+println "+ zstd --memory=1 file"
+zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
+rm file.zst
+println "+ zstd --memory=1K file"
+zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KB file"
+zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KiB file"
+zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1M file"
+zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MB file"
+zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MiB file"
+zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+
+rm file
+exit 0
diff --git a/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact
new file mode 100644
index 00000000000..3785b0f92f3
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact
@@ -0,0 +1,13 @@
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+Should allow numeric parameter without suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
diff --git a/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact
new file mode 100644
index 00000000000..1821648e406
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact
@@ -0,0 +1,13 @@
++ zstd --memory=32LB file
++ zstd --memory=32LiB file
++ zstd --memory=32A file
++ zstd --memory=32r82347dn83 file
++ zstd --memory=32asbdf file
++ zstd --memory=hello file
++ zstd --memory=1 file
++ zstd --memory=1K file
++ zstd --memory=1KB file
++ zstd --memory=1KiB file
++ zstd --memory=1M file
++ zstd --memory=1MB file
++ zstd --memory=1MiB file
diff --git a/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh
new file mode 100755
index 00000000000..a8819d29260
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+println "+ zstd -r * --output-dir-mirror=\"\""
+zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!"
+println "+ zstd -r * --output-dir-flat=\"\""
+zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!"
+exit 0
diff --git a/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact
new file mode 100644
index 00000000000..e12b50427cf
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact
@@ -0,0 +1,2 @@
+error: output dir cannot be empty string (did you mean to pass '.' instead?)
+error: output dir cannot be empty string (did you mean to pass '.' instead?)
diff --git a/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact
new file mode 100644
index 00000000000..1e478cd7531
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact
@@ -0,0 +1,2 @@
++ zstd -r * --output-dir-mirror=""
++ zstd -r * --output-dir-flat=""
diff --git a/3rdparty/zstd/tests/cli-tests/basic/version.sh b/3rdparty/zstd/tests/cli-tests/basic/version.sh
new file mode 100755
index 00000000000..f75eaa84fb4
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/version.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+zstd -V
+zstd --version
diff --git a/3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob
new file mode 100644
index 00000000000..4cc9fb9b234
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob
@@ -0,0 +1,2 @@
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
+*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
diff --git a/3rdparty/zstd/tests/cli-tests/bin/cmp_size b/3rdparty/zstd/tests/cli-tests/bin/cmp_size
new file mode 100755
index 00000000000..8e4bef88eab
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/cmp_size
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -e
+
+usage()
+{
+ printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n"
+}
+
+help()
+{
+ printf "Small utility to compare file sizes without printing them with set -x.\n\n"
+ usage
+}
+
+case "$1" in
+ -h) help; exit 0 ;;
+ --help) help; exit 0 ;;
+esac
+
+if ! test -f $2; then
+ printf "FILE1='%b' is not a file\n\n" "$2"
+ usage
+ exit 1
+fi
+
+if ! test -f $3; then
+ printf "FILE2='%b' is not a file\n\n" "$3"
+ usage
+ exit 1
+fi
+
+
+size1=$(wc -c < $2)
+size2=$(wc -c < $3)
+
+case "$1" in
+ -eq) [ "$size1" -eq "$size2" ] ;;
+ -ne) [ "$size1" -ne "$size2" ] ;;
+ -lt) [ "$size1" -lt "$size2" ] ;;
+ -le) [ "$size1" -le "$size2" ] ;;
+ -gt) [ "$size1" -gt "$size2" ] ;;
+ -ge) [ "$size1" -ge "$size2" ] ;;
+esac
diff --git a/3rdparty/zstd/tests/cli-tests/bin/datagen b/3rdparty/zstd/tests/cli-tests/bin/datagen
new file mode 100755
index 00000000000..8c60cbcc9e7
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/datagen
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"$DATAGEN_BIN" $@
diff --git a/3rdparty/zstd/tests/cli-tests/bin/die b/3rdparty/zstd/tests/cli-tests/bin/die
new file mode 100755
index 00000000000..8633bc975b0
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/die
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+println "${*}" 1>&2
+exit 1
diff --git a/3rdparty/zstd/tests/cli-tests/bin/println b/3rdparty/zstd/tests/cli-tests/bin/println
new file mode 100755
index 00000000000..494eb18c850
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/println
@@ -0,0 +1,2 @@
+#!/bin/sh
+printf '%b\n' "${*}"
diff --git a/3rdparty/zstd/tests/cli-tests/bin/unzstd b/3rdparty/zstd/tests/cli-tests/bin/unzstd
new file mode 120000
index 00000000000..613f917fd7c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/unzstd
@@ -0,0 +1 @@
+zstd \ No newline at end of file
diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstd b/3rdparty/zstd/tests/cli-tests/bin/zstd
new file mode 100755
index 00000000000..7a40aec90d4
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/zstd
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+zstdname=$(basename $0)
+
+if [ -z "$EXEC_PREFIX" ]; then
+ "$ZSTD_SYMLINK_DIR/$zstdname" $@
+else
+ $EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@
+fi
diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstdcat b/3rdparty/zstd/tests/cli-tests/bin/zstdcat
new file mode 120000
index 00000000000..613f917fd7c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/zstdcat
@@ -0,0 +1 @@
+zstd \ No newline at end of file
diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstdgrep b/3rdparty/zstd/tests/cli-tests/bin/zstdgrep
new file mode 100755
index 00000000000..8821ebb5be5
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/zstdgrep
@@ -0,0 +1,2 @@
+#!/bin/sh
+"$ZSTDGREP_BIN" $@
diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstdless b/3rdparty/zstd/tests/cli-tests/bin/zstdless
new file mode 100755
index 00000000000..d1d6f82d20a
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/bin/zstdless
@@ -0,0 +1,2 @@
+#!/bin/sh
+"$ZSTDLESS_BIN" $@
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/setup b/3rdparty/zstd/tests/cli-tests/cltools/setup
new file mode 100755
index 00000000000..3009bd5be5b
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/setup
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+echo "1234" > file
+zstd file
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh
new file mode 100755
index 00000000000..6cd68b7ab27
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+println "+ good path"
+zstdgrep "1234" file file.zst
+println "+ bad path"
+zstdgrep "1234" bad.zst
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit
new file mode 100644
index 00000000000..56a6051ca2b
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit
@@ -0,0 +1 @@
+1 \ No newline at end of file
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact
new file mode 100644
index 00000000000..f147f28d7ea
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact
@@ -0,0 +1 @@
+zstd: can't stat bad.zst : No such file or directory -- ignored
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob
new file mode 100644
index 00000000000..96d4fa2cbcc
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob
@@ -0,0 +1,4 @@
++ good path
+file:1234
+file.zst:1234
++ bad path
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh
new file mode 100755
index 00000000000..a0697bde644
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+println "+ good path"
+zstdless file.zst
+println "+ pass parameters"
+zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd
+println "+ bad path"
+zstdless bad.zst >&2
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact
new file mode 100644
index 00000000000..5a726f1d149
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact
@@ -0,0 +1,2 @@
+zstd: can't stat bad.zst : No such file or directory -- ignored
+bad.zst: No such file or directory
diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob
new file mode 100644
index 00000000000..2784dddf17c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob
@@ -0,0 +1,5 @@
++ good path
+1234
++ pass parameters
+1234
++ bad path
diff --git a/3rdparty/zstd/tests/cli-tests/common/format.sh b/3rdparty/zstd/tests/cli-tests/common/format.sh
new file mode 100644
index 00000000000..e574e973075
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/common/format.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+zstd_supports_format()
+{
+ zstd -h | grep > $INTOVOID -- "--format=$1"
+}
+
+format_extension()
+{
+ if [ "$1" = "zstd" ]; then
+ printf "zst"
+ elif [ "$1" = "gzip" ]; then
+ printf "gz"
+ else
+ printf "$1"
+ fi
+}
diff --git a/3rdparty/zstd/tests/cli-tests/common/mtime.sh b/3rdparty/zstd/tests/cli-tests/common/mtime.sh
new file mode 100644
index 00000000000..344074d398c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/common/mtime.sh
@@ -0,0 +1,13 @@
+. "$COMMON/platform.sh"
+
+MTIME="stat -c %Y"
+case "$UNAME" in
+ Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
+esac
+
+assertSameMTime() {
+ MT1=$($MTIME "$1")
+ MT2=$($MTIME "$2")
+ echo MTIME $MT1 $MT2
+ [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
+}
diff --git a/3rdparty/zstd/tests/cli-tests/common/permissions.sh b/3rdparty/zstd/tests/cli-tests/common/permissions.sh
new file mode 100644
index 00000000000..6bce1f0b387
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/common/permissions.sh
@@ -0,0 +1,18 @@
+. "$COMMON/platform.sh"
+
+GET_PERMS="stat -c %a"
+case "$UNAME" in
+ Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
+esac
+
+assertFilePermissions() {
+ STAT1=$($GET_PERMS "$1")
+ STAT2=$2
+ [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)"
+}
+
+assertSamePermissions() {
+ STAT1=$($GET_PERMS "$1")
+ STAT2=$($GET_PERMS "$2")
+ [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)"
+}
diff --git a/3rdparty/zstd/tests/cli-tests/common/platform.sh b/3rdparty/zstd/tests/cli-tests/common/platform.sh
new file mode 100644
index 00000000000..6eb45eab99e
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/common/platform.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+UNAME=$(uname)
+
+isWindows=false
+INTOVOID="/dev/null"
+case "$UNAME" in
+ GNU) DEVDEVICE="/dev/random" ;;
+ *) DEVDEVICE="/dev/zero" ;;
+esac
+case "$OS" in
+ Windows*)
+ isWindows=true
+ INTOVOID="NUL"
+ DEVDEVICE="NUL"
+ ;;
+esac
+
+case "$UNAME" in
+ Darwin) MD5SUM="md5 -r" ;;
+ FreeBSD) MD5SUM="gmd5sum" ;;
+ NetBSD) MD5SUM="md5 -n" ;;
+ OpenBSD) MD5SUM="md5" ;;
+ *) MD5SUM="md5sum" ;;
+esac
+
+DIFF="diff"
+case "$UNAME" in
+ SunOS) DIFF="gdiff" ;;
+esac
+
+if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
+then
+ hasMT=""
+else
+ hasMT="true"
+fi
diff --git a/3rdparty/zstd/tests/cli-tests/compression/adapt.sh b/3rdparty/zstd/tests/cli-tests/compression/adapt.sh
new file mode 100755
index 00000000000..30b9afaa03b
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/adapt.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+# Test --adapt
+zstd -f file --adapt -c | zstd -t
+
+datagen -g100M > file100M
+
+# Pick parameters to force fast adaptation, even on slow systems
+zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
+
+# Adaption still happens with --no-progress
+zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
diff --git a/3rdparty/zstd/tests/cli-tests/compression/basic.sh b/3rdparty/zstd/tests/cli-tests/compression/basic.sh
new file mode 100755
index 00000000000..950c5a483c5
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/basic.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
+
+# Test compression flags and check that they work
+zstd file ; zstd -t file.zst
+zstd -f file ; zstd -t file.zst
+zstd -f -z file ; zstd -t file.zst
+zstd -f -k file ; zstd -t file.zst
+zstd -f -C file ; zstd -t file.zst
+zstd -f --check file ; zstd -t file.zst
+zstd -f --no-check file ; zstd -t file.zst
+zstd -f -- file ; zstd -t file.zst
+
+# Test output file compression
+zstd -o file-out.zst ; zstd -t file-out.zst
+zstd -fo file-out.zst; zstd -t file-out.zst
+
+# Test compression to stdout
+zstd -c file | zstd -t
+zstd --stdout file | zstd -t
+println bob | zstd | zstd -t
+
+# Test keeping input file when compressing to stdout in gzip mode
+if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
+ $ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file
+ $ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file
+fi
+
+# Test --rm
+cp file file-rm
+zstd --rm file-rm; zstd -t file-rm.zst
+test ! -f file-rm
diff --git a/3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh b/3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh
new file mode 100755
index 00000000000..573481a3f5b
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+# Test --[no-]compress-literals
+zstd file --no-compress-literals -1 -c | zstd -t
+zstd file --no-compress-literals -19 -c | zstd -t
+zstd file --no-compress-literals --fast=1 -c | zstd -t
+zstd file --compress-literals -1 -c | zstd -t
+zstd file --compress-literals --fast=1 -c | zstd -t
diff --git a/3rdparty/zstd/tests/cli-tests/compression/format.sh b/3rdparty/zstd/tests/cli-tests/compression/format.sh
new file mode 100755
index 00000000000..192fa2cf29f
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/format.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "$COMMON/format.sh"
+
+set -e
+
+# Test --format
+zstd --format=zstd file -f
+zstd -t file.zst
+for format in "gzip" "lz4" "xz" "lzma"; do
+ if zstd_supports_format $format; then
+ zstd --format=$format file
+ zstd -t file.$(format_extension $format)
+ zstd -c --format=$format file | zstd -t --format=$format
+ fi
+done
diff --git a/3rdparty/zstd/tests/cli-tests/compression/golden.sh b/3rdparty/zstd/tests/cli-tests/compression/golden.sh
new file mode 100755
index 00000000000..458be9d4aac
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/golden.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+cp -r "$GOLDEN_DIR" golden/
+
+zstd -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/
+
+zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/
+
+# PR #3517 block splitter corruption test
+zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/ \ No newline at end of file
diff --git a/3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh b/3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh
new file mode 100755
index 00000000000..b628b35a0c7
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
+
+# Test gzip specific compression option
+if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
+ $ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+ $ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+
+ # Test -n / --no-name: do not embed original filename in archive
+ $ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+ $ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
+ $ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file
+fi
diff --git a/3rdparty/zstd/tests/cli-tests/compression/levels.sh b/3rdparty/zstd/tests/cli-tests/compression/levels.sh
new file mode 100755
index 00000000000..cc2700a3097
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/levels.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+set -v
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f -q
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst -q
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+
+zstd -5000000000 -f file && die "Level too large, must fail" ||:
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
+
+cmp file-1.zst file-1-env.zst || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact
new file mode 100644
index 00000000000..c8fb79c6896
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -0,0 +1,69 @@
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst -q
+zstd --fast=1 file -o file-f1.zst -q
+zstd -1 file -o file-1.zst -q
+zstd -19 file -o file-19.zst -q
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
+4 files decompressed : 262148 bytes total
+
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f -q
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst -q
+zstd -f file -q
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst -q
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+/*stdin*\ : 65537 bytes
+
+zstd -5000000000 -f file && die "Level too large, must fail" ||:
+error: numeric value overflows 32-bit unsigned int
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
+error: numeric value overflows 32-bit unsigned int
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file -q
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
+Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
+
+cmp file-1.zst file-1-env.zst || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh b/3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh
new file mode 100755
index 00000000000..8f2c61bf75c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Test --long
+zstd -f file --long ; zstd -t file.zst
+zstd -f file --long=20; zstd -t file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh
new file mode 100755
index 00000000000..17a5eb5186b
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Test multi-threaded flags
+zstd --single-thread file -f -q ; zstd -t file.zst
+zstd -T2 -f file -q ; zstd -t file.zst
+zstd --rsyncable -f file -q ; zstd -t file.zst
+zstd -T0 -f file -q ; zstd -t file.zst
+zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
+zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
+
+# multi-thread decompression warning test
+zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
+zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
diff --git a/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
new file mode 100644
index 00000000000..11daff6ba63
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact
@@ -0,0 +1,11 @@
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+file.zst : 65537 bytes
+Warning : decompression does not support multi-threading
+file.zst : 65537 bytes
diff --git a/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh
new file mode 100755
index 00000000000..aeb74cf25eb
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+set -e
+
+# setup
+echo "file1" > file1
+echo "file2" > file2
+
+echo "Test zstd ./file1 - file2"
+rm -f ./file*.zst
+echo "stdin" | zstd ./file1 - ./file2 | zstd -d
+cat file1.zst | zstd -d
+cat file2.zst | zstd -d
+
+echo "Test zstd -d ./file1.zst - file2.zst"
+rm ./file1 ./file2
+echo "stdin" | zstd - | zstd -d ./file1.zst - file2.zst
+cat file1
+cat file2
+
+echo "zstd -d ./file1.zst - file2.zst -c"
+echo "stdin" | zstd | zstd -d ./file1.zst - file2.zst -c
diff --git a/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact
new file mode 100644
index 00000000000..aad61d6359a
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact
@@ -0,0 +1,12 @@
+Test zstd ./file1 - file2
+stdin
+file1
+file2
+Test zstd -d ./file1.zst - file2.zst
+stdin
+file1
+file2
+zstd -d ./file1.zst - file2.zst -c
+file1
+stdin
+file2
diff --git a/3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh b/3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh
new file mode 100755
index 00000000000..5b36017a0ca
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Test --[no-]row-match-finder
+zstd file -7f --row-match-finder
+zstd file -7f --no-row-match-finder
diff --git a/3rdparty/zstd/tests/cli-tests/compression/setup b/3rdparty/zstd/tests/cli-tests/compression/setup
new file mode 100755
index 00000000000..96e2309b6a1
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/setup
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+datagen > file0
+datagen > file1
diff --git a/3rdparty/zstd/tests/cli-tests/compression/stream-size.sh b/3rdparty/zstd/tests/cli-tests/compression/stream-size.sh
new file mode 100755
index 00000000000..7344769a253
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/stream-size.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Test stream size & hint
+datagen -g7654 | zstd --stream-size=7654 | zstd -t
+datagen -g7654 | zstd --size-hint=7000 | zstd -t
diff --git a/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh
new file mode 100755
index 00000000000..88ee11ac84d
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+zstd < file -vv -19 -o file.19.zst
+zstd -vv -l file.19.zst
+
+zstd < file -vv -19 --long -o file.19.long.zst
+zstd -vv -l file.19.long.zst
diff --git a/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob
new file mode 100644
index 00000000000..13534718980
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob
@@ -0,0 +1,5 @@
+...
+*wlog=23*
+...
+*wlog=27*
+...
diff --git a/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob
new file mode 100644
index 00000000000..19913a1679c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob
@@ -0,0 +1,5 @@
+...
+*Window Size: 8388608 B*
+...
+*Window Size: 134217728 B*
+...
diff --git a/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh
new file mode 100755
index 00000000000..e4fe811ce78
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=30 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore
diff --git a/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob
new file mode 100644
index 00000000000..313d216e1e7
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob
@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...
diff --git a/3rdparty/zstd/tests/cli-tests/decompression/golden.sh b/3rdparty/zstd/tests/cli-tests/decompression/golden.sh
new file mode 100755
index 00000000000..36919e69285
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/decompression/golden.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/"
+
+zstd -r -t "$GOLDEN_DIR"
diff --git a/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh
new file mode 100755
index 00000000000..2cab463f840
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh
@@ -0,0 +1,57 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+echo "" > 1
+echo "2" > 2
+echo "23" > 3
+echo "234" > 4
+echo "some data" > file
+
+println "+ passthrough enabled"
+
+zstd file
+
+# Test short files
+zstd -dc --pass-through 1 2 3 4
+
+# Test *cat symlinks
+zstdcat file
+"$ZSTD_SYMLINK_DIR/zcat" file
+"$ZSTD_SYMLINK_DIR/gzcat" file
+
+# Test multiple files with mix of compressed & not
+zstdcat file file.zst
+zstdcat file.zst file
+
+# Test --pass-through
+zstd -dc --pass-through file
+zstd -d --pass-through file -o pass-through-file
+
+# Test legacy implicit passthrough with -fc
+zstd -dcf file
+zstd -dcf file file.zst
+zstd -df < file
+zstd -dcf < file file.zst -
+zstd -dcf < file.zst file -
+
+$DIFF file pass-through-file
+
+println "+ passthrough disabled"
+
+# Test *cat
+zstdcat --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
+# Test zstd without implicit passthrough
+zstd -d file -o no-pass-through-file && die "should fail"
+zstd -d < file && die "should fail"
+
+# Test legacy implicit passthrough with -fc
+zstd --no-pass-through -dcf file && die "should fail"
+zstd --no-pass-through -dcf file file.zst && die "should fail"
+zstd --no-pass-through -df < file && die "should fail"
+zstd --no-pass-through -dcf < file file.zst - && die "should fail"
+zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:
diff --git a/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact
new file mode 100644
index 00000000000..62f96ae420e
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact
@@ -0,0 +1,11 @@
+file :230.00% ( 10 B => 23 B, file.zst)
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: /*stdin*\: unsupported format
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: /*stdin*\: unsupported format
+zstd: /*stdin*\: unsupported format
+zstd: file: unsupported format
diff --git a/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact
new file mode 100644
index 00000000000..b0d494c14aa
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact
@@ -0,0 +1,25 @@
++ passthrough enabled
+
+2
+23
+234
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
++ passthrough disabled
+some data
+some data
+some data
diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh
new file mode 100755
index 00000000000..b500bfebda0
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+for i in $(seq 50); do
+ datagen -s$i > file$i
+done
+touch empty
+
+set -v
+zstd -q --train empty file*
diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact
new file mode 100644
index 00000000000..2747e766cdf
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact
@@ -0,0 +1 @@
+zstd -q --train empty file*
diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh
new file mode 100755
index 00000000000..416b83742f2
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+set -v
+zstd --train
diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit
new file mode 100644
index 00000000000..8351c19397f
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit
@@ -0,0 +1 @@
+14
diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
new file mode 100644
index 00000000000..d7b3ea020f7
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
@@ -0,0 +1,5 @@
+zstd --train
+! Warning : nb of samples too low for proper processing !
+! Please provide _one file per sample_.
+! Alternatively, split files into fixed-size blocks representative of samples, with -B#
+Error 14 : nb of samples too low
diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh
new file mode 100755
index 00000000000..885cac223ad
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+if [ false ]; then
+ for seed in $(seq 100); do
+ datagen -g1000 -s$seed > file$seed
+ done
+
+ zstd --train -r . -o dict0 -qq
+
+ for seed in $(seq 101 200); do
+ datagen -g1000 -s$seed > file$seed
+ done
+
+ zstd --train -r . -o dict1 -qq
+
+ [ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match"
+
+ datagen -g1000 -s0 > file0
+fi
+
+set -v
+zstd files/0 -D dicts/0 -q
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
+zstd -t files/0.zst && die "Must fail" ||:
diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
new file mode 100644
index 00000000000..8896763c1bc
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
@@ -0,0 +1,7 @@
+zstd files/0 -D dicts/0 -q
+zstd -t files/0.zst -D dicts/0
+files/0.zst : 1000 bytes
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
+files/0.zst : Decoding error (36) : Dictionary mismatch
+zstd -t files/0.zst && die "Must fail" ||:
+files/0.zst : Decoding error (36) : Dictionary mismatch
diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh b/3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh
new file mode 100755
index 00000000000..85da2ee538c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/"
+
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst
diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/setup b/3rdparty/zstd/tests/cli-tests/dictionaries/setup
new file mode 100755
index 00000000000..616c73eb804
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dictionaries/setup
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+cp -r ../files .
+cp -r ../dicts .
diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/setup_once b/3rdparty/zstd/tests/cli-tests/dictionaries/setup_once
new file mode 100755
index 00000000000..1241c578214
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/dictionaries/setup_once
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+
+mkdir files/ dicts/
+
+for seed in $(seq 50); do
+ datagen -g1000 -s$seed > files/$seed
+done
+
+zstd --train -r files -o dicts/0 -qq
+
+for seed in $(seq 51 100); do
+ datagen -g1000 -s$seed > files/$seed
+done
+
+zstd --train -r files -o dicts/1 -qq
+
+cmp dicts/0 dicts/1 && die "dictionaries must not match!"
+
+datagen -g1000 > files/0
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh
new file mode 100755
index 00000000000..1aa4525fb7d
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# motivated by issue #3523
+
+datagen > file
+mkdir out
+chmod 000 out
+
+zstd file -q --trace-file-stat -o out/file.zst
+zstd -tq out/file.zst
+
+chmod 777 out
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact
new file mode 100644
index 00000000000..95deaf2b1e8
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact
@@ -0,0 +1,26 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, out/file.zst)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(-1, out/file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(out/file.zst)
+Trace:FileStat: > UTIL_stat(-1, out/file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+zstd: out/file.zst: Permission denied
+zstd: can't stat out/file.zst : Permission denied -- ignored
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh
new file mode 100755
index 00000000000..c5f5900318f
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+chmod 642 file
+
+zstd file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
new file mode 100644
index 00000000000..32d248ee50b
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
@@ -0,0 +1,42 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file, file.zst)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_setFileStat(4, file.zst)
+Trace:FileStat: > UTIL_stat(4, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_chmod(file.zst, 0642)
+Trace:FileStat: > fchmod
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_utime(file.zst)
+Trace:FileStat: < 0
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh
new file mode 100755
index 00000000000..99ebfc46f59
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 00000000000..a25a355723c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,24 @@
+Trace:FileStat: > UTIL_isLink(file)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 65537
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 65537
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh
new file mode 100755
index 00000000000..83794619860
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -q --trace-file-stat -o file.zst
+zstd -tq file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 00000000000..9183c467459
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,24 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file.zst)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < -1
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
new file mode 100755
index 00000000000..64f4b036216
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen > file
+
+zstd < file -cq --trace-file-stat > file.zst
+zstd -tq file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 00000000000..66b630e433c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,18 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < -1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < -1
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh
new file mode 100755
index 00000000000..9e68f8f332a
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+chmod 642 file.zst
+
+zstd -dq --trace-file-stat file.zst
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
new file mode 100644
index 00000000000..ad3a0deb868
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
@@ -0,0 +1,38 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isSameFile(file.zst, file)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_setFileStat(4, file)
+Trace:FileStat: > UTIL_stat(4, file)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_chmod(file, 0642)
+Trace:FileStat: > fchmod
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_utime(file)
+Trace:FileStat: < 0
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
new file mode 100755
index 00000000000..518c2a950d9
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat file.zst > file
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
new file mode 100644
index 00000000000..8b60063c5fc
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
@@ -0,0 +1,18 @@
+Trace:FileStat: > UTIL_isLink(file.zst)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(file.zst)
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isDirectoryStat()
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_stat(-1, file.zst)
+Trace:FileStat: < 1
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
new file mode 100755
index 00000000000..135d75524d0
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst -o file
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
new file mode 100644
index 00000000000..716a7ac7b34
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
@@ -0,0 +1,20 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(file)
+Trace:FileStat: > UTIL_stat(-1, file)
+Trace:FileStat: < 1
+Trace:FileStat: < 1
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
new file mode 100755
index 00000000000..495f07b17fe
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+datagen | zstd -q > file.zst
+
+zstd -dcq --trace-file-stat < file.zst > file
diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
new file mode 100644
index 00000000000..2f6412059fc
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
@@ -0,0 +1,14 @@
+Trace:FileStat: > UTIL_isConsole(0)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(1)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isConsole(2)
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
+Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
+Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
+Trace:FileStat: < 0
+Trace:FileStat: < 0
diff --git a/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh
new file mode 100755
index 00000000000..708878f2643
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+println >&2 "Tests cases where progress information should not be printed"
+
+for args in \
+ "" \
+ "--fake-stderr-is-console -q" \
+ "--fake-stderr-is-console -qq --progress" \
+ "--no-progress --fake-stderr-is-console" \
+ "--no-progress --fake-stderr-is-console -v"
+do
+ println >&2 "args = $args"
+ println >&2 "compress file to file"
+ zstd $args -f hello
+ println >&2 "compress pipe to pipe"
+ zstd $args < hello > $INTOVOID
+ println >&2 "compress pipe to file"
+ zstd $args < hello -fo hello.zst
+ println >&2 "compress file to pipe"
+ zstd $args hello -c > $INTOVOID
+ println >&2 "compress 2 files"
+ zstd $args -f hello world
+
+ println >&2 "decompress file to file"
+ zstd $args -d -f hello.zst
+ println >&2 "decompress pipe to pipe"
+ zstd $args -d < hello.zst > $INTOVOID
+ println >&2 "decompress pipe to file"
+ zstd $args -d < hello.zst -fo hello
+ println >&2 "decompress file to pipe"
+ zstd $args -d hello.zst -c > $INTOVOID
+ println >&2 "decompress 2 files"
+ zstd $args -d -f hello.zst world.zst
+ println >&2 ""
+done
diff --git a/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob
new file mode 100644
index 00000000000..d0f91122769
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob
@@ -0,0 +1,96 @@
+Tests cases where progress information should not be printed
+args =
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --fake-stderr-is-console -q
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -qq --progress
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --no-progress --fake-stderr-is-console
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --no-progress --fake-stderr-is-console -v
+compress file to file
+*Zstandard CLI*
+hello*hello.zst*
+compress pipe to pipe
+*Zstandard CLI*
+*stdin*stdout*
+compress pipe to file
+*Zstandard CLI*
+*stdin*hello.zst*
+compress file to pipe
+*Zstandard CLI*
+*hello*stdout*
+compress 2 files
+*Zstandard CLI*
+*hello*hello.zst*
+*world*world.zst*
+2 files compressed*
+decompress file to file
+*Zstandard CLI*
+hello.zst*
+decompress pipe to pipe
+*Zstandard CLI*
+*stdin*
+decompress pipe to file
+*Zstandard CLI*
+*stdin*
+decompress file to pipe
+*Zstandard CLI*
+hello.zst*
+decompress 2 files
+*Zstandard CLI*
+hello.zst*
+world.zst*
+2 files decompressed*
diff --git a/3rdparty/zstd/tests/cli-tests/progress/progress.sh b/3rdparty/zstd/tests/cli-tests/progress/progress.sh
new file mode 100755
index 00000000000..eb464993a24
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/progress/progress.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+println >&2 "Tests cases where progress information should be printed"
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+for args in \
+ "--progress" \
+ "--fake-stderr-is-console" \
+ "--progress --fake-stderr-is-console -q"; do
+ println >&2 "args = $args"
+ println >&2 "compress file to file"
+ zstd $args -f hello
+ println >&2 "compress pipe to pipe"
+ zstd $args < hello > $INTOVOID
+ println >&2 "compress pipe to file"
+ zstd $args < hello -fo hello.zst
+ println >&2 "compress file to pipe"
+ zstd $args hello -c > $INTOVOID
+ println >&2 "compress 2 files"
+ zstd $args -f hello world
+
+ println >&2 "decompress file to file"
+ zstd $args -d -f hello.zst
+ println >&2 "decompress pipe to pipe"
+ zstd $args -d < hello.zst > $INTOVOID
+ println >&2 "decompress pipe to file"
+ zstd $args -d < hello.zst -fo hello
+ println >&2 "decompress file to pipe"
+ zstd $args -d hello.zst -c > $INTOVOID
+ println >&2 "decompress 2 files"
+ zstd $args -d -f hello.zst world.zst
+ println >&2 ""
+done
diff --git a/3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob b/3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob
new file mode 100644
index 00000000000..ca620d3c282
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob
@@ -0,0 +1,62 @@
+Tests cases where progress information should be printed
+args = --progress
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --fake-stderr-is-console
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --progress --fake-stderr-is-console -q
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
diff --git a/3rdparty/zstd/tests/cli-tests/run.py b/3rdparty/zstd/tests/cli-tests/run.py
new file mode 100755
index 00000000000..46564d2ffcb
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/run.py
@@ -0,0 +1,731 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ##########################################################################
+
+import argparse
+import contextlib
+import copy
+import fnmatch
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import typing
+
+
+ZSTD_SYMLINKS = [
+ "zstd",
+ "zstdmt",
+ "unzstd",
+ "zstdcat",
+ "zcat",
+ "gzip",
+ "gunzip",
+ "gzcat",
+ "lzma",
+ "unlzma",
+ "xz",
+ "unxz",
+ "lz4",
+ "unlz4",
+]
+
+
+EXCLUDED_DIRS = {
+ "bin",
+ "common",
+ "scratch",
+}
+
+
+EXCLUDED_BASENAMES = {
+ "setup",
+ "setup_once",
+ "teardown",
+ "teardown_once",
+ "README.md",
+ "run.py",
+ ".gitignore",
+}
+
+EXCLUDED_SUFFIXES = [
+ ".exact",
+ ".glob",
+ ".ignore",
+ ".exit",
+]
+
+
+def exclude_dir(dirname: str) -> bool:
+ """
+ Should files under the directory :dirname: be excluded from the test runner?
+ """
+ if dirname in EXCLUDED_DIRS:
+ return True
+ return False
+
+
+def exclude_file(filename: str) -> bool:
+ """Should the file :filename: be excluded from the test runner?"""
+ if filename in EXCLUDED_BASENAMES:
+ return True
+ for suffix in EXCLUDED_SUFFIXES:
+ if filename.endswith(suffix):
+ return True
+ return False
+
+def read_file(filename: str) -> bytes:
+ """Reads the file :filename: and returns the contents as bytes."""
+ with open(filename, "rb") as f:
+ return f.read()
+
+
+def diff(a: bytes, b: bytes) -> str:
+ """Returns a diff between two different byte-strings :a: and :b:."""
+ assert a != b
+ with tempfile.NamedTemporaryFile("wb") as fa:
+ fa.write(a)
+ fa.flush()
+ with tempfile.NamedTemporaryFile("wb") as fb:
+ fb.write(b)
+ fb.flush()
+
+ diff_bytes = subprocess.run(["diff", fa.name, fb.name], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout
+ return diff_bytes.decode("utf8")
+
+
+def pop_line(data: bytes) -> typing.Tuple[typing.Optional[bytes], bytes]:
+ """
+ Pop the first line from :data: and returns the first line and the remainder
+ of the data as a tuple. If :data: is empty, returns :(None, data):. Otherwise
+ the first line always ends in a :\n:, even if it is the last line and :data:
+ doesn't end in :\n:.
+ """
+ NEWLINE = b"\n"
+
+ if data == b'':
+ return (None, data)
+
+ parts = data.split(NEWLINE, maxsplit=1)
+ line = parts[0] + NEWLINE
+ if len(parts) == 1:
+ return line, b''
+
+ return line, parts[1]
+
+
+def glob_line_matches(actual: bytes, expect: bytes) -> bool:
+ """
+ Does the `actual` line match the expected glob line `expect`?
+ """
+ return fnmatch.fnmatchcase(actual.strip(), expect.strip())
+
+
+def glob_diff(actual: bytes, expect: bytes) -> bytes:
+ """
+ Returns None if the :actual: content matches the expected glob :expect:,
+ otherwise returns the diff bytes.
+ """
+ diff = b''
+ actual_line, actual = pop_line(actual)
+ expect_line, expect = pop_line(expect)
+ while True:
+ # Handle end of file conditions - allow extra newlines
+ while expect_line is None and actual_line == b"\n":
+ actual_line, actual = pop_line(actual)
+ while actual_line is None and expect_line == b"\n":
+ expect_line, expect = pop_line(expect)
+
+ if expect_line is None and actual_line is None:
+ if diff == b'':
+ return None
+ return diff
+ elif expect_line is None:
+ diff += b"---\n"
+ while actual_line != None:
+ diff += b"> "
+ diff += actual_line
+ actual_line, actual = pop_line(actual)
+ return diff
+ elif actual_line is None:
+ diff += b"---\n"
+ while expect_line != None:
+ diff += b"< "
+ diff += expect_line
+ expect_line, expect = pop_line(expect)
+ return diff
+
+ assert expect_line is not None
+ assert actual_line is not None
+
+ if expect_line == b'...\n':
+ next_expect_line, expect = pop_line(expect)
+ if next_expect_line is None:
+ if diff == b'':
+ return None
+ return diff
+ while not glob_line_matches(actual_line, next_expect_line):
+ actual_line, actual = pop_line(actual)
+ if actual_line is None:
+ diff += b"---\n"
+ diff += b"< "
+ diff += next_expect_line
+ return diff
+ expect_line = next_expect_line
+ continue
+
+ if not glob_line_matches(actual_line, expect_line):
+ diff += b'---\n'
+ diff += b'< ' + expect_line
+ diff += b'> ' + actual_line
+
+ actual_line, actual = pop_line(actual)
+ expect_line, expect = pop_line(expect)
+
+
+class Options:
+ """Options configuring how to run a :TestCase:."""
+ def __init__(
+ self,
+ env: typing.Dict[str, str],
+ timeout: typing.Optional[int],
+ verbose: bool,
+ preserve: bool,
+ scratch_dir: str,
+ test_dir: str,
+ set_exact_output: bool,
+ ) -> None:
+ self.env = env
+ self.timeout = timeout
+ self.verbose = verbose
+ self.preserve = preserve
+ self.scratch_dir = scratch_dir
+ self.test_dir = test_dir
+ self.set_exact_output = set_exact_output
+
+
+class TestCase:
+ """
+ Logic and state related to running a single test case.
+
+ 1. Initialize the test case.
+ 2. Launch the test case with :TestCase.launch():.
+ This will start the test execution in a subprocess, but
+ not wait for completion. So you could launch multiple test
+ cases in parallel. This will now print any test output.
+ 3. Analyze the results with :TestCase.analyze():. This will
+ join the test subprocess, check the results against the
+ expectations, and print the results to stdout.
+
+ :TestCase.run(): is also provided which combines the launch & analyze
+ steps for single-threaded use-cases.
+
+ All other methods, prefixed with _, are private helper functions.
+ """
+ def __init__(self, test_filename: str, options: Options) -> None:
+ """
+ Initialize the :TestCase: for the test located in :test_filename:
+ with the given :options:.
+ """
+ self._opts = options
+ self._test_file = test_filename
+ self._test_name = os.path.normpath(
+ os.path.relpath(test_filename, start=self._opts.test_dir)
+ )
+ self._success = {}
+ self._message = {}
+ self._test_stdin = None
+ self._scratch_dir = os.path.abspath(os.path.join(self._opts.scratch_dir, self._test_name))
+
+ @property
+ def name(self) -> str:
+ """Returns the unique name for the test."""
+ return self._test_name
+
+ def launch(self) -> None:
+ """
+ Launch the test case as a subprocess, but do not block on completion.
+ This allows users to run multiple tests in parallel. Results aren't yet
+ printed out.
+ """
+ self._launch_test()
+
+ def analyze(self) -> bool:
+ """
+ Must be called after :TestCase.launch():. Joins the test subprocess and
+ checks the results against expectations. Finally prints the results to
+ stdout and returns the success.
+ """
+ self._join_test()
+ self._check_exit()
+ self._check_stderr()
+ self._check_stdout()
+ self._analyze_results()
+ return self._succeeded
+
+ def run(self) -> bool:
+ """Shorthand for combining both :TestCase.launch(): and :TestCase.analyze():."""
+ self.launch()
+ return self.analyze()
+
+ def _log(self, *args, **kwargs) -> None:
+ """Logs test output."""
+ print(file=sys.stdout, *args, **kwargs)
+
+ def _vlog(self, *args, **kwargs) -> None:
+ """Logs verbose test output."""
+ if self._opts.verbose:
+ print(file=sys.stdout, *args, **kwargs)
+
+ def _test_environment(self) -> typing.Dict[str, str]:
+ """
+ Returns the environment to be used for the
+ test subprocess.
+ """
+ # We want to omit ZSTD cli flags so tests will be consistent across environments
+ env = {k: v for k, v in os.environ.items() if not k.startswith("ZSTD")}
+ for k, v in self._opts.env.items():
+ self._vlog(f"${k}='{v}'")
+ env[k] = v
+ return env
+
+ def _launch_test(self) -> None:
+ """Launch the test subprocess, but do not join it."""
+ args = [os.path.abspath(self._test_file)]
+ stdin_name = f"{self._test_file}.stdin"
+ if os.path.exists(stdin_name):
+ self._test_stdin = open(stdin_name, "rb")
+ stdin = self._test_stdin
+ else:
+ stdin = subprocess.DEVNULL
+ cwd = self._scratch_dir
+ env = self._test_environment()
+ self._test_process = subprocess.Popen(
+ args=args,
+ stdin=stdin,
+ cwd=cwd,
+ env=env,
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE
+ )
+
+ def _join_test(self) -> None:
+ """Join the test process and save stderr, stdout, and the exit code."""
+ (stdout, stderr) = self._test_process.communicate(timeout=self._opts.timeout)
+ self._output = {}
+ self._output["stdout"] = stdout
+ self._output["stderr"] = stderr
+ self._exit_code = self._test_process.returncode
+ self._test_process = None
+ if self._test_stdin is not None:
+ self._test_stdin.close()
+ self._test_stdin = None
+
+ def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None:
+ """
+ Check the output named :out_name: for an exact match against the :expected: content.
+ Saves the success and message.
+ """
+ check_name = f"check_{out_name}"
+ actual = self._output[out_name]
+ if actual == expected:
+ self._success[check_name] = True
+ self._message[check_name] = f"{out_name} matches!"
+ else:
+ self._success[check_name] = False
+ self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}"
+
+ if self._opts.set_exact_output:
+ with open(exact_name, "wb") as f:
+ f.write(actual)
+
+ def _check_output_glob(self, out_name: str, expected: bytes) -> None:
+ """
+ Check the output named :out_name: for a glob match against the :expected: glob.
+ Saves the success and message.
+ """
+ check_name = f"check_{out_name}"
+ actual = self._output[out_name]
+ diff = glob_diff(actual, expected)
+ if diff is None:
+ self._success[check_name] = True
+ self._message[check_name] = f"{out_name} matches!"
+ else:
+ utf8_diff = diff.decode('utf8')
+ self._success[check_name] = False
+ self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{utf8_diff}"
+
+ def _check_output(self, out_name: str) -> None:
+ """
+ Checks the output named :out_name: for a match against the expectation.
+ We check for a .exact, .glob, and a .ignore file. If none are found we
+ expect that the output should be empty.
+
+ If :Options.preserve: was set then we save the scratch directory and
+ save the stderr, stdout, and exit code to the scratch directory for
+ debugging.
+ """
+ if self._opts.preserve:
+ # Save the output to the scratch directory
+ actual_name = os.path.join(self._scratch_dir, f"{out_name}")
+ with open(actual_name, "wb") as f:
+ f.write(self._output[out_name])
+
+ exact_name = f"{self._test_file}.{out_name}.exact"
+ glob_name = f"{self._test_file}.{out_name}.glob"
+ ignore_name = f"{self._test_file}.{out_name}.ignore"
+
+ if os.path.exists(exact_name):
+ return self._check_output_exact(out_name, read_file(exact_name), exact_name)
+ elif os.path.exists(glob_name):
+ return self._check_output_glob(out_name, read_file(glob_name))
+ else:
+ check_name = f"check_{out_name}"
+ self._success[check_name] = True
+ self._message[check_name] = f"{out_name} ignored!"
+
+ def _check_stderr(self) -> None:
+ """Checks the stderr output against the expectation."""
+ self._check_output("stderr")
+
+ def _check_stdout(self) -> None:
+ """Checks the stdout output against the expectation."""
+ self._check_output("stdout")
+
+ def _check_exit(self) -> None:
+ """
+ Checks the exit code against expectations. If a .exit file
+ exists, we expect that the exit code matches the contents.
+ Otherwise we expect the exit code to be zero.
+
+ If :Options.preserve: is set we save the exit code to the
+ scratch directory under the filename "exit".
+ """
+ if self._opts.preserve:
+ exit_name = os.path.join(self._scratch_dir, "exit")
+ with open(exit_name, "w") as f:
+ f.write(str(self._exit_code) + "\n")
+ exit_name = f"{self._test_file}.exit"
+ if os.path.exists(exit_name):
+ exit_code: int = int(read_file(exit_name))
+ else:
+ exit_code: int = 0
+ if exit_code == self._exit_code:
+ self._success["check_exit"] = True
+ self._message["check_exit"] = "Exit code matches!"
+ else:
+ self._success["check_exit"] = False
+ self._message["check_exit"] = f"Exit code mismatch! Expected {exit_code} but got {self._exit_code}"
+
+ def _analyze_results(self) -> None:
+ """
+ After all tests have been checked, collect all the successes
+ and messages, and print the results to stdout.
+ """
+ STATUS = {True: "PASS", False: "FAIL"}
+ checks = sorted(self._success.keys())
+ self._succeeded = all(self._success.values())
+ self._log(f"{STATUS[self._succeeded]}: {self._test_name}")
+
+ if not self._succeeded or self._opts.verbose:
+ for check in checks:
+ if self._opts.verbose or not self._success[check]:
+ self._log(f"{STATUS[self._success[check]]}: {self._test_name}.{check}")
+ self._log(self._message[check])
+
+ self._log("----------------------------------------")
+
+
+class TestSuite:
+ """
+ Setup & teardown test suite & cases.
+ This class is intended to be used as a context manager.
+
+ TODO: Make setup/teardown failure emit messages, not throw exceptions.
+ """
+ def __init__(self, test_directory: str, options: Options) -> None:
+ self._opts = options
+ self._test_dir = os.path.abspath(test_directory)
+ rel_test_dir = os.path.relpath(test_directory, start=self._opts.test_dir)
+ assert not rel_test_dir.startswith(os.path.sep)
+ self._scratch_dir = os.path.normpath(os.path.join(self._opts.scratch_dir, rel_test_dir))
+
+ def __enter__(self) -> 'TestSuite':
+ self._setup_once()
+ return self
+
+ def __exit__(self, _exc_type, _exc_value, _traceback) -> None:
+ self._teardown_once()
+
+ @contextlib.contextmanager
+ def test_case(self, test_basename: str) -> TestCase:
+ """
+ Context manager for a test case in the test suite.
+ Pass the basename of the test relative to the :test_directory:.
+ """
+ assert os.path.dirname(test_basename) == ""
+ try:
+ self._setup(test_basename)
+ test_filename = os.path.join(self._test_dir, test_basename)
+ yield TestCase(test_filename, self._opts)
+ finally:
+ self._teardown(test_basename)
+
+ def _remove_scratch_dir(self, dir: str) -> None:
+ """Helper to remove a scratch directory with sanity checks"""
+ assert "scratch" in dir
+ assert dir.startswith(self._scratch_dir)
+ assert os.path.exists(dir)
+ shutil.rmtree(dir)
+
+ def _setup_once(self) -> None:
+ if os.path.exists(self._scratch_dir):
+ self._remove_scratch_dir(self._scratch_dir)
+ os.makedirs(self._scratch_dir)
+ setup_script = os.path.join(self._test_dir, "setup_once")
+ if os.path.exists(setup_script):
+ self._run_script(setup_script, cwd=self._scratch_dir)
+
+ def _teardown_once(self) -> None:
+ assert os.path.exists(self._scratch_dir)
+ teardown_script = os.path.join(self._test_dir, "teardown_once")
+ if os.path.exists(teardown_script):
+ self._run_script(teardown_script, cwd=self._scratch_dir)
+ if not self._opts.preserve:
+ self._remove_scratch_dir(self._scratch_dir)
+
+ def _setup(self, test_basename: str) -> None:
+ test_scratch_dir = os.path.join(self._scratch_dir, test_basename)
+ assert not os.path.exists(test_scratch_dir)
+ os.makedirs(test_scratch_dir)
+ setup_script = os.path.join(self._test_dir, "setup")
+ if os.path.exists(setup_script):
+ self._run_script(setup_script, cwd=test_scratch_dir)
+
+ def _teardown(self, test_basename: str) -> None:
+ test_scratch_dir = os.path.join(self._scratch_dir, test_basename)
+ assert os.path.exists(test_scratch_dir)
+ teardown_script = os.path.join(self._test_dir, "teardown")
+ if os.path.exists(teardown_script):
+ self._run_script(teardown_script, cwd=test_scratch_dir)
+ if not self._opts.preserve:
+ self._remove_scratch_dir(test_scratch_dir)
+
+ def _run_script(self, script: str, cwd: str) -> None:
+ env = copy.copy(os.environ)
+ for k, v in self._opts.env.items():
+ env[k] = v
+ try:
+ subprocess.run(
+ args=[script],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=cwd,
+ env=env,
+ check=True,
+ )
+ except subprocess.CalledProcessError as e:
+ print(f"{script} failed with exit code {e.returncode}!")
+ print(f"stderr:\n{e.stderr}")
+ print(f"stdout:\n{e.stdout}")
+ raise
+
+TestSuites = typing.Dict[str, typing.List[str]]
+
+def get_all_tests(options: Options) -> TestSuites:
+ """
+ Find all the test in the test directory and return the test suites.
+ """
+ test_suites = {}
+ for root, dirs, files in os.walk(options.test_dir, topdown=True):
+ dirs[:] = [d for d in dirs if not exclude_dir(d)]
+ test_cases = []
+ for file in files:
+ if not exclude_file(file):
+ test_cases.append(file)
+ assert root == os.path.normpath(root)
+ test_suites[root] = test_cases
+ return test_suites
+
+
+def resolve_listed_tests(
+ tests: typing.List[str], options: Options
+) -> TestSuites:
+ """
+ Resolve the list of tests passed on the command line into their
+ respective test suites. Tests can either be paths, or test names
+ relative to the test directory.
+ """
+ test_suites = {}
+ for test in tests:
+ if not os.path.exists(test):
+ test = os.path.join(options.test_dir, test)
+ if not os.path.exists(test):
+ raise RuntimeError(f"Test {test} does not exist!")
+
+ test = os.path.normpath(os.path.abspath(test))
+ assert test.startswith(options.test_dir)
+ test_suite = os.path.dirname(test)
+ test_case = os.path.basename(test)
+ test_suites.setdefault(test_suite, []).append(test_case)
+
+ return test_suites
+
+def run_tests(test_suites: TestSuites, options: Options) -> bool:
+ """
+ Runs all the test in the :test_suites: with the given :options:.
+ Prints the results to stdout.
+ """
+ tests = {}
+ for test_dir, test_files in test_suites.items():
+ with TestSuite(test_dir, options) as test_suite:
+ test_files = sorted(set(test_files))
+ for test_file in test_files:
+ with test_suite.test_case(test_file) as test_case:
+ tests[test_case.name] = test_case.run()
+
+ successes = 0
+ for test, status in tests.items():
+ if status:
+ successes += 1
+ else:
+ print(f"FAIL: {test}")
+ if successes == len(tests):
+ print(f"PASSED all {len(tests)} tests!")
+ return True
+ else:
+ print(f"FAILED {len(tests) - successes} / {len(tests)} tests!")
+ return False
+
+
+def setup_zstd_symlink_dir(zstd_symlink_dir: str, zstd: str) -> None:
+ assert os.path.join("bin", "symlinks") in zstd_symlink_dir
+ if not os.path.exists(zstd_symlink_dir):
+ os.makedirs(zstd_symlink_dir)
+ for symlink in ZSTD_SYMLINKS:
+ path = os.path.join(zstd_symlink_dir, symlink)
+ if os.path.exists(path):
+ os.remove(path)
+ os.symlink(zstd, path)
+
+if __name__ == "__main__":
+ CLI_TEST_DIR = os.path.dirname(sys.argv[0])
+ REPO_DIR = os.path.join(CLI_TEST_DIR, "..", "..")
+ PROGRAMS_DIR = os.path.join(REPO_DIR, "programs")
+ TESTS_DIR = os.path.join(REPO_DIR, "tests")
+ ZSTD_PATH = os.path.join(PROGRAMS_DIR, "zstd")
+ ZSTDGREP_PATH = os.path.join(PROGRAMS_DIR, "zstdgrep")
+ ZSTDLESS_PATH = os.path.join(PROGRAMS_DIR, "zstdless")
+ DATAGEN_PATH = os.path.join(TESTS_DIR, "datagen")
+
+ parser = argparse.ArgumentParser(
+ (
+ "Runs the zstd CLI tests. Exits nonzero on failure. Default arguments are\n"
+ "generally correct. Pass --preserve to preserve test output for debugging,\n"
+ "and --verbose to get verbose test output.\n"
+ )
+ )
+ parser.add_argument(
+ "--preserve",
+ action="store_true",
+ help="Preserve the scratch directory TEST_DIR/scratch/ for debugging purposes."
+ )
+ parser.add_argument("--verbose", action="store_true", help="Verbose test output.")
+ parser.add_argument("--timeout", default=200, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.")
+ parser.add_argument(
+ "--exec-prefix",
+ default=None,
+ help="Sets the EXEC_PREFIX environment variable. Prefix to invocations of the zstd CLI."
+ )
+ parser.add_argument(
+ "--zstd",
+ default=ZSTD_PATH,
+ help="Sets the ZSTD_BIN environment variable. Path of the zstd CLI."
+ )
+ parser.add_argument(
+ "--zstdgrep",
+ default=ZSTDGREP_PATH,
+ help="Sets the ZSTDGREP_BIN environment variable. Path of the zstdgrep CLI."
+ )
+ parser.add_argument(
+ "--zstdless",
+ default=ZSTDLESS_PATH,
+ help="Sets the ZSTDLESS_BIN environment variable. Path of the zstdless CLI."
+ )
+ parser.add_argument(
+ "--datagen",
+ default=DATAGEN_PATH,
+ help="Sets the DATAGEN_BIN environment variable. Path to the datagen CLI."
+ )
+ parser.add_argument(
+ "--test-dir",
+ default=CLI_TEST_DIR,
+ help=(
+ "Runs the tests under this directory. "
+ "Adds TEST_DIR/bin/ to path. "
+ "Scratch directory located in TEST_DIR/scratch/."
+ )
+ )
+ parser.add_argument(
+ "--set-exact-output",
+ action="store_true",
+ help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists"
+ )
+ parser.add_argument(
+ "tests",
+ nargs="*",
+ help="Run only these test cases. Can either be paths or test names relative to TEST_DIR/"
+ )
+ args = parser.parse_args()
+
+ if args.timeout <= 0:
+ args.timeout = None
+
+ args.test_dir = os.path.normpath(os.path.abspath(args.test_dir))
+ bin_dir = os.path.abspath(os.path.join(args.test_dir, "bin"))
+ zstd_symlink_dir = os.path.join(bin_dir, "symlinks")
+ scratch_dir = os.path.join(args.test_dir, "scratch")
+
+ setup_zstd_symlink_dir(zstd_symlink_dir, os.path.abspath(args.zstd))
+
+ env = {}
+ if args.exec_prefix is not None:
+ env["EXEC_PREFIX"] = args.exec_prefix
+ env["ZSTD_SYMLINK_DIR"] = zstd_symlink_dir
+ env["ZSTD_REPO_DIR"] = os.path.abspath(REPO_DIR)
+ env["DATAGEN_BIN"] = os.path.abspath(args.datagen)
+ env["ZSTDGREP_BIN"] = os.path.abspath(args.zstdgrep)
+ env["ZSTDLESS_BIN"] = os.path.abspath(args.zstdless)
+ env["COMMON"] = os.path.abspath(os.path.join(args.test_dir, "common"))
+ env["PATH"] = bin_dir + ":" + os.getenv("PATH", "")
+ env["LC_ALL"] = "C"
+
+ opts = Options(
+ env=env,
+ timeout=args.timeout,
+ verbose=args.verbose,
+ preserve=args.preserve,
+ test_dir=args.test_dir,
+ scratch_dir=scratch_dir,
+ set_exact_output=args.set_exact_output,
+ )
+
+ if len(args.tests) == 0:
+ tests = get_all_tests(opts)
+ else:
+ tests = resolve_listed_tests(args.tests, opts)
+
+ success = run_tests(tests, opts)
+ if success:
+ sys.exit(0)
+ else:
+ sys.exit(1)
diff --git a/3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup
new file mode 100755
index 00000000000..cf391ed2117
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup
@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+println "hello" > hello
+println "world" > world
+zstd hello world
diff --git a/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh
new file mode 100755
index 00000000000..74ec063d13c
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+
+# Test zstdcat symlink in bin/
+zstdcat hello.zst
+zstdcat hello.zst world
+zstdcat hello world.zst
+zstdcat hello.zst world.zst
+
+# Test local zstdcat symlink
+ln -s $(which zstd) ./zstdcat
+./zstdcat hello.zst
diff --git a/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact
new file mode 100644
index 00000000000..3205b059b34
--- /dev/null
+++ b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact
@@ -0,0 +1,8 @@
+hello
+hello
+world
+hello
+world
+hello
+world
+hello