diff options
Diffstat (limited to '3rdparty/zstd/tests/cli-tests')
95 files changed, 2439 insertions, 0 deletions
diff --git a/3rdparty/zstd/tests/cli-tests/.gitignore b/3rdparty/zstd/tests/cli-tests/.gitignore new file mode 100644 index 00000000000..0ad01b24eff --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/.gitignore @@ -0,0 +1,6 @@ +!bin/ +!datagen +!zstdcat + +scratch/ +bin/symlinks diff --git a/3rdparty/zstd/tests/cli-tests/README.md b/3rdparty/zstd/tests/cli-tests/README.md new file mode 100644 index 00000000000..7ca07c3e9ac --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/README.md @@ -0,0 +1,258 @@ +# CLI tests + +The CLI tests are focused on testing the zstd CLI. +They are intended to be simple tests that the CLI and arguments work as advertised. +They are not intended to test the library, only the code in `programs/`. +The library will get incidental coverage, but if you find yourself trying to trigger a specific condition in the library, this is the wrong tool. + +## Test runner usage + +The test runner `run.py` will run tests against the in-tree build of `zstd` and `datagen` by default. Which means that `zstd` and `datagen` must be built. + +The `zstd` binary used can be passed with `--zstd /path/to/zstd`. +Additionally, to run `zstd` through a tool like `valgrind` or `qemu`, set the `--exec-prefix 'valgrind -q'` flag. + +Similarly, the `--datagen`, and `--zstdgrep` flags can be set to specify +the paths to their respective binaries. However, these tools do not use +the `EXEC_PREFIX`. + +Each test executes in its own scratch directory under `scratch/test/name`. E.g. `scratch/basic/help.sh/`. Normally these directories are removed after the test executes. However, the `--preserve` flag will preserve these directories after execution, and save the tests exit code, stdout, and stderr in the scratch directory to `exit`, `stderr`, and `stdout` respectively. This can be useful for debugging/editing a test and updating the expected output. + +### Running all the tests + +By default the test runner `run.py` will run all the tests, and report the results. + +Examples: + +``` +./run.py +./run.py --preserve +./run.py --zstd ../../build/programs/zstd --datagen ../../build/tests/datagen +``` + +### Running specific tests + +A set of test names can be passed to the test runner `run.py` to only execute those tests. +This can be useful for writing or debugging a test, especially with `--preserve`. + +The test name can either be the path to the test file, or the test name, which is the path relative to the test directory. + +Examples: + +``` +./run.py basic/help.sh +./run.py --preserve basic/help.sh basic/version.sh +./run.py --preserve --verbose basic/help.sh +``` + +### Updating exact output + +If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written. + +Example: +``` +./run.py --set-exact-output +./run.py basic/help.sh --set-exact-output +``` + +## Writing a test + +Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts. +After the script executes, the exit code, stderr, and stdout are compared against the expectations. + +Each test is run in a clean directory that the test can use for intermediate files. This directory will be cleaned up at the end of the test, unless `--preserve` is passed to the test runner. Additionally, the `setup` script can prepare the directory before the test runs. + +### Calling zstd, utilities, and environment variables + +The `$PATH` for tests is prepended with the `bin/` sub-directory, which contains helper scripts for ease of testing. +The `zstd` binary will call the zstd binary specified by `run.py` with the correct `$EXEC_PREFIX`. +Similarly, `datagen`, `unzstd`, `zstdgrep`, `zstdcat`, etc, are provided. + +Helper utilities like `cmp_size`, `println`, and `die` are provided here too. See their scripts for details. + +Common shell script libraries are provided under `common/`, with helper variables and functions. They can be sourced with `source "$COMMON/library.sh`. + +Lastly, environment variables are provided for testing, which can be listed when calling `run.py` with `--verbose`. +They are generally used by the helper scripts in `bin/` to coordinate everything. + +### Basic test case + +When executing your `$TEST` executable, by default the exit code is expected to be `0`. However, you can provide an alternate expected exit code in a `$TEST.exit` file. + +When executing your `$TEST` executable, by default the expected stderr and stdout are empty. However, you can override the default by providing one of three files: + +* `$TEST.{stdout,stderr}.exact` +* `$TEST.{stdout,stderr}.glob` +* `$TEST.{stdout,stderr}.ignore` + +If you provide a `.exact` file, the output is expected to exactly match, byte-for-byte. + +If you provide a `.glob` file, the output is expected to match the expected file, where each line is interpreted as a glob syntax. Additionally, a line containing only `...` matches all lines until the next expected line matches. + +If you provide a `.ignore` file, the output is ignored. + +#### Passing examples + +All these examples pass. + +Exit 1, and change the expectation to be 1. + +``` +exit-1.sh +--- +#!/bin/sh +exit 1 +--- + +exit-1.sh.exit +--- +1 +--- +``` + +Check the stdout output exactly matches. + +``` +echo.sh +--- +#!/bin/sh +echo "hello world" +--- + +echo.sh.stdout.exact +--- +hello world +--- +``` + +Check the stderr output using a glob. + +``` +random.sh +--- +#!/bin/sh +head -c 10 < /dev/urandom | xxd >&2 +--- + +random.sh.stderr.glob +--- +00000000: * * * * * * +``` + +Multiple lines can be matched with ... + +``` +random-num-lines.sh +--- +#!/bin/sh +echo hello +seq 0 $RANDOM +echo world +--- + +random-num-lines.sh.stdout.glob +--- +hello +0 +... +world +--- +``` + +#### Failing examples + +Exit code is expected to be 0, but is 1. + +``` +exit-1.sh +--- +#!/bin/sh +exit 1 +--- +``` + +Stdout is expected to be empty, but isn't. + +``` +echo.sh +--- +#!/bin/sh +echo hello world +``` + +Stderr is expected to be hello but is world. + +``` +hello.sh +--- +#!/bin/sh +echo world >&2 +--- + +hello.sh.stderr.exact +--- +hello +--- +``` + +### Setup & teardown scripts + +Finally, test writing can be eased with setup and teardown scripts. +Each directory in the test directory is a test-suite consisting of all tests within that directory (but not sub-directories). +This test suite can come with 4 scripts to help test writing: + +* `setup_once` +* `teardown_once` +* `setup` +* `teardown` + +The `setup_once` and `teardown_once` are run once before and after all the tests in the suite respectively. +They operate in the scratch directory for the test suite, which is the parent directory of each scratch directory for each test case. +They can do work that is shared between tests to improve test efficiency. +For example, the `dictionaries/setup_once` script builds several dictionaries, for use in the `dictionaries` tests. + +The `setup` and `teardown` scripts run before and after each test case respectively, in the test case's scratch directory. +These scripts can do work that is shared between test cases to make tests more succinct. +For example, the `dictionaries/setup` script copies the dictionaries built by the `dictionaries/setup_once` script into the test's scratch directory, to make them easier to use, and make sure they aren't accidentally modified. + +#### Examples + +``` +basic/setup +--- +#!/bin/sh +# Create some files for testing with +datagen > file +datagen > file0 +datagen > file1 +--- + +basic/test.sh +--- +#!/bin/sh +zstd file file0 file1 +--- + +dictionaries/setup_once +--- +#!/bin/sh +set -e + +mkdir files/ dicts/ +for i in $(seq 10); do + datagen -g1000 > files/$i +done + +zstd --train -r files/ -o dicts/0 +--- + +dictionaries/setup +--- +#!/bin/sh + +# Runs in the test case's scratch directory. +# The test suite's scratch directory that +# `setup_once` operates in is the parent directory. +cp -r ../files ../dicts . +--- +``` diff --git a/3rdparty/zstd/tests/cli-tests/basic/help.sh b/3rdparty/zstd/tests/cli-tests/basic/help.sh new file mode 100755 index 00000000000..927c3ffbcc5 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/help.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +println "+ zstd -h" +zstd -h +println "+ zstd -H" +zstd -H +println "+ zstd --help" +zstd --help diff --git a/3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob new file mode 100644 index 00000000000..21bc28c3fa8 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/help.sh.stdout.glob @@ -0,0 +1,34 @@ ++ zstd -h +Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided. + +Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT* + +Options: + -o OUTPUT Write output to a single file, OUTPUT. + -k, --keep Preserve INPUT file(s). *Default* + --rm Remove INPUT file(s) after successful (de)compression. + + -# Desired compression level, where `#` is a number between 1 and 19; + lower numbers provide faster compression, higher numbers yield + better compression ratios. *Default: 3* + + -d, --decompress Perform decompression. + -D DICT Use DICT as the dictionary for compression or decompression. + + -f, --force Disable input and output checks. Allows overwriting existing files, + receiving input from the console, printing output to STDOUT, and + operating on links, block devices, etc. Unrecognized formats will be + passed-through through as-is. + + -h Display short usage and exit. + -H, --help Display full help and exit. + -V, --version Display the program version and exit. + ++ zstd -H +... +Advanced options: +... ++ zstd --help +... +Advanced options: +... diff --git a/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh new file mode 100755 index 00000000000..88d734d0d9d --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +echo "some data" > file + +println "+ zstd --memory=32LB file" +zstd --memory=32LB file && die "Should not allow bogus suffix" +println "+ zstd --memory=32LiB file" +zstd --memory=32LiB file && die "Should not allow bogus suffix" +println "+ zstd --memory=32A file" +zstd --memory=32A file && die "Should not allow bogus suffix" +println "+ zstd --memory=32r82347dn83 file" +zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix" +println "+ zstd --memory=32asbdf file" +zstd --memory=32asbdf file && die "Should not allow bogus suffix" +println "+ zstd --memory=hello file" +zstd --memory=hello file && die "Should not allow non-numeric parameter" +println "+ zstd --memory=1 file" +zstd -q --memory=1 file && die "Should allow numeric parameter without suffix" +rm file.zst +println "+ zstd --memory=1K file" +zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix" +rm file.zst +println "+ zstd --memory=1KB file" +zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix" +rm file.zst +println "+ zstd --memory=1KiB file" +zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix" +rm file.zst +println "+ zstd --memory=1M file" +zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix" +rm file.zst +println "+ zstd --memory=1MB file" +zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix" +rm file.zst +println "+ zstd --memory=1MiB file" +zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix" +rm file.zst + +rm file +exit 0 diff --git a/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact new file mode 100644 index 00000000000..3785b0f92f3 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact @@ -0,0 +1,13 @@ +error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed +error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed +error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed +error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed +error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed +error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed +Should allow numeric parameter without suffix +Should allow numeric parameter with expected suffix +Should allow numeric parameter with expected suffix +Should allow numeric parameter with expected suffix +Should allow numeric parameter with expected suffix +Should allow numeric parameter with expected suffix +Should allow numeric parameter with expected suffix diff --git a/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact new file mode 100644 index 00000000000..1821648e406 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact @@ -0,0 +1,13 @@ ++ zstd --memory=32LB file ++ zstd --memory=32LiB file ++ zstd --memory=32A file ++ zstd --memory=32r82347dn83 file ++ zstd --memory=32asbdf file ++ zstd --memory=hello file ++ zstd --memory=1 file ++ zstd --memory=1K file ++ zstd --memory=1KB file ++ zstd --memory=1KiB file ++ zstd --memory=1M file ++ zstd --memory=1MB file ++ zstd --memory=1MiB file diff --git a/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh new file mode 100755 index 00000000000..a8819d29260 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +println "+ zstd -r * --output-dir-mirror=\"\"" +zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!" +println "+ zstd -r * --output-dir-flat=\"\"" +zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!" +exit 0 diff --git a/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact new file mode 100644 index 00000000000..e12b50427cf --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact @@ -0,0 +1,2 @@ +error: output dir cannot be empty string (did you mean to pass '.' instead?) +error: output dir cannot be empty string (did you mean to pass '.' instead?) diff --git a/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact new file mode 100644 index 00000000000..1e478cd7531 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact @@ -0,0 +1,2 @@ ++ zstd -r * --output-dir-mirror="" ++ zstd -r * --output-dir-flat="" diff --git a/3rdparty/zstd/tests/cli-tests/basic/version.sh b/3rdparty/zstd/tests/cli-tests/basic/version.sh new file mode 100755 index 00000000000..f75eaa84fb4 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/version.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +zstd -V +zstd --version diff --git a/3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob new file mode 100644 index 00000000000..4cc9fb9b234 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/basic/version.sh.stdout.glob @@ -0,0 +1,2 @@ +*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet *** +*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet *** diff --git a/3rdparty/zstd/tests/cli-tests/bin/cmp_size b/3rdparty/zstd/tests/cli-tests/bin/cmp_size new file mode 100755 index 00000000000..8e4bef88eab --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/cmp_size @@ -0,0 +1,44 @@ +#!/bin/sh + +set -e + +usage() +{ + printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n" +} + +help() +{ + printf "Small utility to compare file sizes without printing them with set -x.\n\n" + usage +} + +case "$1" in + -h) help; exit 0 ;; + --help) help; exit 0 ;; +esac + +if ! test -f $2; then + printf "FILE1='%b' is not a file\n\n" "$2" + usage + exit 1 +fi + +if ! test -f $3; then + printf "FILE2='%b' is not a file\n\n" "$3" + usage + exit 1 +fi + + +size1=$(wc -c < $2) +size2=$(wc -c < $3) + +case "$1" in + -eq) [ "$size1" -eq "$size2" ] ;; + -ne) [ "$size1" -ne "$size2" ] ;; + -lt) [ "$size1" -lt "$size2" ] ;; + -le) [ "$size1" -le "$size2" ] ;; + -gt) [ "$size1" -gt "$size2" ] ;; + -ge) [ "$size1" -ge "$size2" ] ;; +esac diff --git a/3rdparty/zstd/tests/cli-tests/bin/datagen b/3rdparty/zstd/tests/cli-tests/bin/datagen new file mode 100755 index 00000000000..8c60cbcc9e7 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/datagen @@ -0,0 +1,3 @@ +#!/bin/sh + +"$DATAGEN_BIN" $@ diff --git a/3rdparty/zstd/tests/cli-tests/bin/die b/3rdparty/zstd/tests/cli-tests/bin/die new file mode 100755 index 00000000000..8633bc975b0 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/die @@ -0,0 +1,4 @@ +#!/bin/sh + +println "${*}" 1>&2 +exit 1 diff --git a/3rdparty/zstd/tests/cli-tests/bin/println b/3rdparty/zstd/tests/cli-tests/bin/println new file mode 100755 index 00000000000..494eb18c850 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/println @@ -0,0 +1,2 @@ +#!/bin/sh +printf '%b\n' "${*}" diff --git a/3rdparty/zstd/tests/cli-tests/bin/unzstd b/3rdparty/zstd/tests/cli-tests/bin/unzstd new file mode 120000 index 00000000000..613f917fd7c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/unzstd @@ -0,0 +1 @@ +zstd
\ No newline at end of file diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstd b/3rdparty/zstd/tests/cli-tests/bin/zstd new file mode 100755 index 00000000000..7a40aec90d4 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/zstd @@ -0,0 +1,9 @@ +#!/bin/sh + +zstdname=$(basename $0) + +if [ -z "$EXEC_PREFIX" ]; then + "$ZSTD_SYMLINK_DIR/$zstdname" $@ +else + $EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@ +fi diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstdcat b/3rdparty/zstd/tests/cli-tests/bin/zstdcat new file mode 120000 index 00000000000..613f917fd7c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/zstdcat @@ -0,0 +1 @@ +zstd
\ No newline at end of file diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstdgrep b/3rdparty/zstd/tests/cli-tests/bin/zstdgrep new file mode 100755 index 00000000000..8821ebb5be5 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/zstdgrep @@ -0,0 +1,2 @@ +#!/bin/sh +"$ZSTDGREP_BIN" $@ diff --git a/3rdparty/zstd/tests/cli-tests/bin/zstdless b/3rdparty/zstd/tests/cli-tests/bin/zstdless new file mode 100755 index 00000000000..d1d6f82d20a --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/bin/zstdless @@ -0,0 +1,2 @@ +#!/bin/sh +"$ZSTDLESS_BIN" $@ diff --git a/3rdparty/zstd/tests/cli-tests/cltools/setup b/3rdparty/zstd/tests/cli-tests/cltools/setup new file mode 100755 index 00000000000..3009bd5be5b --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/setup @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +echo "1234" > file +zstd file diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh new file mode 100755 index 00000000000..6cd68b7ab27 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +println "+ good path" +zstdgrep "1234" file file.zst +println "+ bad path" +zstdgrep "1234" bad.zst diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit @@ -0,0 +1 @@ +1
\ No newline at end of file diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact new file mode 100644 index 00000000000..f147f28d7ea --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact @@ -0,0 +1 @@ +zstd: can't stat bad.zst : No such file or directory -- ignored diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob new file mode 100644 index 00000000000..96d4fa2cbcc --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob @@ -0,0 +1,4 @@ ++ good path +file:1234 +file.zst:1234 ++ bad path diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh new file mode 100755 index 00000000000..a0697bde644 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +println "+ good path" +zstdless file.zst +println "+ pass parameters" +zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd +println "+ bad path" +zstdless bad.zst >&2 diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact new file mode 100644 index 00000000000..5a726f1d149 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact @@ -0,0 +1,2 @@ +zstd: can't stat bad.zst : No such file or directory -- ignored +bad.zst: No such file or directory diff --git a/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob new file mode 100644 index 00000000000..2784dddf17c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob @@ -0,0 +1,5 @@ ++ good path +1234 ++ pass parameters +1234 ++ bad path diff --git a/3rdparty/zstd/tests/cli-tests/common/format.sh b/3rdparty/zstd/tests/cli-tests/common/format.sh new file mode 100644 index 00000000000..e574e973075 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/common/format.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "$COMMON/platform.sh" + +zstd_supports_format() +{ + zstd -h | grep > $INTOVOID -- "--format=$1" +} + +format_extension() +{ + if [ "$1" = "zstd" ]; then + printf "zst" + elif [ "$1" = "gzip" ]; then + printf "gz" + else + printf "$1" + fi +} diff --git a/3rdparty/zstd/tests/cli-tests/common/mtime.sh b/3rdparty/zstd/tests/cli-tests/common/mtime.sh new file mode 100644 index 00000000000..344074d398c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/common/mtime.sh @@ -0,0 +1,13 @@ +. "$COMMON/platform.sh" + +MTIME="stat -c %Y" +case "$UNAME" in + Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; +esac + +assertSameMTime() { + MT1=$($MTIME "$1") + MT2=$($MTIME "$2") + echo MTIME $MT1 $MT2 + [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" +} diff --git a/3rdparty/zstd/tests/cli-tests/common/permissions.sh b/3rdparty/zstd/tests/cli-tests/common/permissions.sh new file mode 100644 index 00000000000..6bce1f0b387 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/common/permissions.sh @@ -0,0 +1,18 @@ +. "$COMMON/platform.sh" + +GET_PERMS="stat -c %a" +case "$UNAME" in + Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; +esac + +assertFilePermissions() { + STAT1=$($GET_PERMS "$1") + STAT2=$2 + [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" +} + +assertSamePermissions() { + STAT1=$($GET_PERMS "$1") + STAT2=$($GET_PERMS "$2") + [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" +} diff --git a/3rdparty/zstd/tests/cli-tests/common/platform.sh b/3rdparty/zstd/tests/cli-tests/common/platform.sh new file mode 100644 index 00000000000..6eb45eab99e --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/common/platform.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +UNAME=$(uname) + +isWindows=false +INTOVOID="/dev/null" +case "$UNAME" in + GNU) DEVDEVICE="/dev/random" ;; + *) DEVDEVICE="/dev/zero" ;; +esac +case "$OS" in + Windows*) + isWindows=true + INTOVOID="NUL" + DEVDEVICE="NUL" + ;; +esac + +case "$UNAME" in + Darwin) MD5SUM="md5 -r" ;; + FreeBSD) MD5SUM="gmd5sum" ;; + NetBSD) MD5SUM="md5 -n" ;; + OpenBSD) MD5SUM="md5" ;; + *) MD5SUM="md5sum" ;; +esac + +DIFF="diff" +case "$UNAME" in + SunOS) DIFF="gdiff" ;; +esac + +if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' +then + hasMT="" +else + hasMT="true" +fi diff --git a/3rdparty/zstd/tests/cli-tests/compression/adapt.sh b/3rdparty/zstd/tests/cli-tests/compression/adapt.sh new file mode 100755 index 00000000000..30b9afaa03b --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/adapt.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e + +# Test --adapt +zstd -f file --adapt -c | zstd -t + +datagen -g100M > file100M + +# Pick parameters to force fast adaptation, even on slow systems +zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression" + +# Adaption still happens with --no-progress +zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression" diff --git a/3rdparty/zstd/tests/cli-tests/compression/basic.sh b/3rdparty/zstd/tests/cli-tests/compression/basic.sh new file mode 100755 index 00000000000..950c5a483c5 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/basic.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -e + +# Uncomment the set -v line for debugging +# set -v + +# Test compression flags and check that they work +zstd file ; zstd -t file.zst +zstd -f file ; zstd -t file.zst +zstd -f -z file ; zstd -t file.zst +zstd -f -k file ; zstd -t file.zst +zstd -f -C file ; zstd -t file.zst +zstd -f --check file ; zstd -t file.zst +zstd -f --no-check file ; zstd -t file.zst +zstd -f -- file ; zstd -t file.zst + +# Test output file compression +zstd -o file-out.zst ; zstd -t file-out.zst +zstd -fo file-out.zst; zstd -t file-out.zst + +# Test compression to stdout +zstd -c file | zstd -t +zstd --stdout file | zstd -t +println bob | zstd | zstd -t + +# Test keeping input file when compressing to stdout in gzip mode +if $(command -v $ZSTD_SYMLINK_DIR/gzip); then + $ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file + $ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file +fi + +# Test --rm +cp file file-rm +zstd --rm file-rm; zstd -t file-rm.zst +test ! -f file-rm diff --git a/3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh b/3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh new file mode 100755 index 00000000000..573481a3f5b --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/compress-literals.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +# Test --[no-]compress-literals +zstd file --no-compress-literals -1 -c | zstd -t +zstd file --no-compress-literals -19 -c | zstd -t +zstd file --no-compress-literals --fast=1 -c | zstd -t +zstd file --compress-literals -1 -c | zstd -t +zstd file --compress-literals --fast=1 -c | zstd -t diff --git a/3rdparty/zstd/tests/cli-tests/compression/format.sh b/3rdparty/zstd/tests/cli-tests/compression/format.sh new file mode 100755 index 00000000000..192fa2cf29f --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/format.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "$COMMON/format.sh" + +set -e + +# Test --format +zstd --format=zstd file -f +zstd -t file.zst +for format in "gzip" "lz4" "xz" "lzma"; do + if zstd_supports_format $format; then + zstd --format=$format file + zstd -t file.$(format_extension $format) + zstd -c --format=$format file | zstd -t --format=$format + fi +done diff --git a/3rdparty/zstd/tests/cli-tests/compression/golden.sh b/3rdparty/zstd/tests/cli-tests/compression/golden.sh new file mode 100755 index 00000000000..458be9d4aac --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/golden.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/" +cp -r "$GOLDEN_DIR" golden/ + +zstd -rf golden/ --output-dir-mirror golden-compressed/ +zstd -r -t golden-compressed/ + +zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/ +zstd -r -t golden-compressed/ + +# PR #3517 block splitter corruption test +zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/ +zstd -r -t golden-compressed/
\ No newline at end of file diff --git a/3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh b/3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh new file mode 100755 index 00000000000..b628b35a0c7 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/gzip-compat.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +# Uncomment the set -v line for debugging +# set -v + +# Test gzip specific compression option +if $(command -v $ZSTD_SYMLINK_DIR/gzip); then + $ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz + $ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz + + # Test -n / --no-name: do not embed original filename in archive + $ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz + $ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz + $ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file +fi diff --git a/3rdparty/zstd/tests/cli-tests/compression/levels.sh b/3rdparty/zstd/tests/cli-tests/compression/levels.sh new file mode 100755 index 00000000000..cc2700a3097 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/levels.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +set -e +set -v + +datagen > file + +# Compress with various levels and ensure that their sizes are ordered +zstd --fast=10 file -o file-f10.zst -q +zstd --fast=1 file -o file-f1.zst -q +zstd -1 file -o file-1.zst -q +zstd -19 file -o file-19.zst -q + +zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst + +cmp_size -lt file-19.zst file-1.zst +cmp_size -lt file-1.zst file-f1.zst +cmp_size -lt file-f1.zst file-f10.zst + +# Test default levels +zstd --fast file -f -q +cmp file.zst file-f1.zst || die "--fast is not level -1" + +zstd -0 file -o file-0.zst -q +zstd -f file -q +cmp file.zst file-0.zst || die "Level 0 is not the default level" + +# Test level clamping +zstd -99 file -o file-99.zst -q +cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19" +zstd --fast=200000 file -c | zstd -t + +zstd -5000000000 -f file && die "Level too large, must fail" ||: +zstd --fast=5000000000 -f file && die "Level too large, must fail" ||: + +# Test setting a level through the environment variable +ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q +ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q +ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q +ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q + +cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level" +cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level" +cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level" +cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level" + +# Test invalid environment clevel is the default level +zstd -f file -q +ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst + +# Test environment clevel is overridden by command line +ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q +ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q + +cmp file-1.zst file-1-env.zst || die "Environment variable not overridden" +cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden" diff --git a/3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact new file mode 100644 index 00000000000..c8fb79c6896 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/levels.sh.stderr.exact @@ -0,0 +1,69 @@ + +datagen > file + +# Compress with various levels and ensure that their sizes are ordered +zstd --fast=10 file -o file-f10.zst -q +zstd --fast=1 file -o file-f1.zst -q +zstd -1 file -o file-1.zst -q +zstd -19 file -o file-19.zst -q + +zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst +4 files decompressed : 262148 bytes total + +cmp_size -lt file-19.zst file-1.zst +cmp_size -lt file-1.zst file-f1.zst +cmp_size -lt file-f1.zst file-f10.zst + +# Test default levels +zstd --fast file -f -q +cmp file.zst file-f1.zst || die "--fast is not level -1" + +zstd -0 file -o file-0.zst -q +zstd -f file -q +cmp file.zst file-0.zst || die "Level 0 is not the default level" + +# Test level clamping +zstd -99 file -o file-99.zst -q +cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19" +zstd --fast=200000 file -c | zstd -t +/*stdin*\ : 65537 bytes + +zstd -5000000000 -f file && die "Level too large, must fail" ||: +error: numeric value overflows 32-bit unsigned int +zstd --fast=5000000000 -f file && die "Level too large, must fail" ||: +error: numeric value overflows 32-bit unsigned int + +# Test setting a level through the environment variable +ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q +ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q +ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q +ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q + +cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level" +cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level" +cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level" +cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level" + +# Test invalid environment clevel is the default level +zstd -f file -q +ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value +ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value +ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value +ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value +ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value +ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value +ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst +Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large + +# Test environment clevel is overridden by command line +ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q +ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q + +cmp file-1.zst file-1-env.zst || die "Environment variable not overridden" +cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden" diff --git a/3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh b/3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh new file mode 100755 index 00000000000..8f2c61bf75c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/long-distance-matcher.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +# Test --long +zstd -f file --long ; zstd -t file.zst +zstd -f file --long=20; zstd -t file.zst diff --git a/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh new file mode 100755 index 00000000000..17a5eb5186b --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +# Test multi-threaded flags +zstd --single-thread file -f -q ; zstd -t file.zst +zstd -T2 -f file -q ; zstd -t file.zst +zstd --rsyncable -f file -q ; zstd -t file.zst +zstd -T0 -f file -q ; zstd -t file.zst +zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst +zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst + +# multi-thread decompression warning test +zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 +zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4 diff --git a/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact new file mode 100644 index 00000000000..11daff6ba63 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact @@ -0,0 +1,11 @@ +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +file.zst : 65537 bytes +Warning : decompression does not support multi-threading +file.zst : 65537 bytes diff --git a/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh new file mode 100755 index 00000000000..aeb74cf25eb --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +# setup +echo "file1" > file1 +echo "file2" > file2 + +echo "Test zstd ./file1 - file2" +rm -f ./file*.zst +echo "stdin" | zstd ./file1 - ./file2 | zstd -d +cat file1.zst | zstd -d +cat file2.zst | zstd -d + +echo "Test zstd -d ./file1.zst - file2.zst" +rm ./file1 ./file2 +echo "stdin" | zstd - | zstd -d ./file1.zst - file2.zst +cat file1 +cat file2 + +echo "zstd -d ./file1.zst - file2.zst -c" +echo "stdin" | zstd | zstd -d ./file1.zst - file2.zst -c diff --git a/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact new file mode 100644 index 00000000000..aad61d6359a --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact @@ -0,0 +1,12 @@ +Test zstd ./file1 - file2 +stdin +file1 +file2 +Test zstd -d ./file1.zst - file2.zst +stdin +file1 +file2 +zstd -d ./file1.zst - file2.zst -c +file1 +stdin +file2 diff --git a/3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh b/3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh new file mode 100755 index 00000000000..5b36017a0ca --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/row-match-finder.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +# Test --[no-]row-match-finder +zstd file -7f --row-match-finder +zstd file -7f --no-row-match-finder diff --git a/3rdparty/zstd/tests/cli-tests/compression/setup b/3rdparty/zstd/tests/cli-tests/compression/setup new file mode 100755 index 00000000000..96e2309b6a1 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/setup @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +datagen > file +datagen > file0 +datagen > file1 diff --git a/3rdparty/zstd/tests/cli-tests/compression/stream-size.sh b/3rdparty/zstd/tests/cli-tests/compression/stream-size.sh new file mode 100755 index 00000000000..7344769a253 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/stream-size.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +# Test stream size & hint +datagen -g7654 | zstd --stream-size=7654 | zstd -t +datagen -g7654 | zstd --size-hint=7000 | zstd -t diff --git a/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh new file mode 100755 index 00000000000..88ee11ac84d --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +. "$COMMON/platform.sh" + +zstd < file -vv -19 -o file.19.zst +zstd -vv -l file.19.zst + +zstd < file -vv -19 --long -o file.19.long.zst +zstd -vv -l file.19.long.zst diff --git a/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob new file mode 100644 index 00000000000..13534718980 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob @@ -0,0 +1,5 @@ +... +*wlog=23* +... +*wlog=27* +... diff --git a/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob new file mode 100644 index 00000000000..19913a1679c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob @@ -0,0 +1,5 @@ +... +*Window Size: 8388608 B* +... +*Window Size: 134217728 B* +... diff --git a/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh new file mode 100755 index 00000000000..e4fe811ce78 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh @@ -0,0 +1,9 @@ +#!/bin/sh +datagen -g1G > file +zstd --long=30 -1 --single-thread --no-content-size -f file +zstd -l -v file.zst + +# We want to ignore stderr (its outputting "*** zstd command line interface +# 64-bits v1.5.3, by Yann Collet ***") + +rm file file.zst diff --git a/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore diff --git a/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob new file mode 100644 index 00000000000..313d216e1e7 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob @@ -0,0 +1,3 @@ +... +Window Size: 1.000 GiB (1073741824 B) +... diff --git a/3rdparty/zstd/tests/cli-tests/decompression/golden.sh b/3rdparty/zstd/tests/cli-tests/decompression/golden.sh new file mode 100755 index 00000000000..36919e69285 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/decompression/golden.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/" + +zstd -r -t "$GOLDEN_DIR" diff --git a/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh new file mode 100755 index 00000000000..2cab463f840 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +set -e + +. "$COMMON/platform.sh" + +echo "" > 1 +echo "2" > 2 +echo "23" > 3 +echo "234" > 4 +echo "some data" > file + +println "+ passthrough enabled" + +zstd file + +# Test short files +zstd -dc --pass-through 1 2 3 4 + +# Test *cat symlinks +zstdcat file +"$ZSTD_SYMLINK_DIR/zcat" file +"$ZSTD_SYMLINK_DIR/gzcat" file + +# Test multiple files with mix of compressed & not +zstdcat file file.zst +zstdcat file.zst file + +# Test --pass-through +zstd -dc --pass-through file +zstd -d --pass-through file -o pass-through-file + +# Test legacy implicit passthrough with -fc +zstd -dcf file +zstd -dcf file file.zst +zstd -df < file +zstd -dcf < file file.zst - +zstd -dcf < file.zst file - + +$DIFF file pass-through-file + +println "+ passthrough disabled" + +# Test *cat +zstdcat --no-pass-through file && die "should fail" +"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail" +"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail" +# Test zstd without implicit passthrough +zstd -d file -o no-pass-through-file && die "should fail" +zstd -d < file && die "should fail" + +# Test legacy implicit passthrough with -fc +zstd --no-pass-through -dcf file && die "should fail" +zstd --no-pass-through -dcf file file.zst && die "should fail" +zstd --no-pass-through -df < file && die "should fail" +zstd --no-pass-through -dcf < file file.zst - && die "should fail" +zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||: diff --git a/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact new file mode 100644 index 00000000000..62f96ae420e --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact @@ -0,0 +1,11 @@ +file :230.00% ( 10 B => 23 B, file.zst) +zstd: file: unsupported format +zstd: file: unsupported format +zstd: file: unsupported format +zstd: file: unsupported format +zstd: /*stdin*\: unsupported format +zstd: file: unsupported format +zstd: file: unsupported format +zstd: /*stdin*\: unsupported format +zstd: /*stdin*\: unsupported format +zstd: file: unsupported format diff --git a/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact new file mode 100644 index 00000000000..b0d494c14aa --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact @@ -0,0 +1,25 @@ ++ passthrough enabled + +2 +23 +234 +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data ++ passthrough disabled +some data +some data +some data diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh new file mode 100755 index 00000000000..b500bfebda0 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e +for i in $(seq 50); do + datagen -s$i > file$i +done +touch empty + +set -v +zstd -q --train empty file* diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact new file mode 100644 index 00000000000..2747e766cdf --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact @@ -0,0 +1 @@ +zstd -q --train empty file* diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh new file mode 100755 index 00000000000..416b83742f2 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -v +zstd --train diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit new file mode 100644 index 00000000000..8351c19397f --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit @@ -0,0 +1 @@ +14 diff --git a/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact new file mode 100644 index 00000000000..d7b3ea020f7 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact @@ -0,0 +1,5 @@ +zstd --train +! Warning : nb of samples too low for proper processing ! +! Please provide _one file per sample_. +! Alternatively, split files into fixed-size blocks representative of samples, with -B# +Error 14 : nb of samples too low diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh new file mode 100755 index 00000000000..885cac223ad --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. "$COMMON/platform.sh" + +set -e + +if [ false ]; then + for seed in $(seq 100); do + datagen -g1000 -s$seed > file$seed + done + + zstd --train -r . -o dict0 -qq + + for seed in $(seq 101 200); do + datagen -g1000 -s$seed > file$seed + done + + zstd --train -r . -o dict1 -qq + + [ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match" + + datagen -g1000 -s0 > file0 +fi + +set -v +zstd files/0 -D dicts/0 -q +zstd -t files/0.zst -D dicts/0 +zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: +zstd -t files/0.zst && die "Must fail" ||: diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact new file mode 100644 index 00000000000..8896763c1bc --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact @@ -0,0 +1,7 @@ +zstd files/0 -D dicts/0 -q +zstd -t files/0.zst -D dicts/0 +files/0.zst : 1000 bytes +zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: +files/0.zst : Decoding error (36) : Dictionary mismatch +zstd -t files/0.zst && die "Must fail" ||: +files/0.zst : Decoding error (36) : Dictionary mismatch diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh b/3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh new file mode 100755 index 00000000000..85da2ee538c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dictionaries/golden.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/" +GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/" + +zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst +zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/setup b/3rdparty/zstd/tests/cli-tests/dictionaries/setup new file mode 100755 index 00000000000..616c73eb804 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dictionaries/setup @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +cp -r ../files . +cp -r ../dicts . diff --git a/3rdparty/zstd/tests/cli-tests/dictionaries/setup_once b/3rdparty/zstd/tests/cli-tests/dictionaries/setup_once new file mode 100755 index 00000000000..1241c578214 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/dictionaries/setup_once @@ -0,0 +1,24 @@ +#!/bin/sh + +set -e + +. "$COMMON/platform.sh" + + +mkdir files/ dicts/ + +for seed in $(seq 50); do + datagen -g1000 -s$seed > files/$seed +done + +zstd --train -r files -o dicts/0 -qq + +for seed in $(seq 51 100); do + datagen -g1000 -s$seed > files/$seed +done + +zstd --train -r files -o dicts/1 -qq + +cmp dicts/0 dicts/1 && die "dictionaries must not match!" + +datagen -g1000 > files/0 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh new file mode 100755 index 00000000000..1aa4525fb7d --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# motivated by issue #3523 + +datagen > file +mkdir out +chmod 000 out + +zstd file -q --trace-file-stat -o out/file.zst +zstd -tq out/file.zst + +chmod 777 out diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact new file mode 100644 index 00000000000..95deaf2b1e8 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact @@ -0,0 +1,26 @@ +Trace:FileStat: > UTIL_isLink(file) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 65537 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isDirectoryStat() +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isSameFile(file, out/file.zst) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_stat(-1, out/file.zst) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(out/file.zst) +Trace:FileStat: > UTIL_stat(-1, out/file.zst) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +zstd: out/file.zst: Permission denied +zstd: can't stat out/file.zst : Permission denied -- ignored diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh new file mode 100755 index 00000000000..c5f5900318f --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +datagen > file +chmod 642 file + +zstd file -q --trace-file-stat -o file.zst +zstd -tq file.zst diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact new file mode 100644 index 00000000000..32d248ee50b --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact @@ -0,0 +1,42 @@ +Trace:FileStat: > UTIL_isLink(file) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 65537 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isDirectoryStat() +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isSameFile(file, file.zst) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file.zst) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file.zst) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_getFileSize(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 65537 +Trace:FileStat: > UTIL_setFileStat(4, file.zst) +Trace:FileStat: > UTIL_stat(4, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_chmod(file.zst, 0642) +Trace:FileStat: > fchmod +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_utime(file.zst) +Trace:FileStat: < 0 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh new file mode 100755 index 00000000000..99ebfc46f59 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +datagen > file + +zstd file -cq --trace-file-stat > file.zst +zstd -tq file.zst diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact new file mode 100644 index 00000000000..a25a355723c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact @@ -0,0 +1,24 @@ +Trace:FileStat: > UTIL_isLink(file) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(1) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 65537 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isDirectoryStat() +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) +Trace:FileStat: > UTIL_stat(-1, /*stdout*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 65537 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh new file mode 100755 index 00000000000..83794619860 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +datagen > file + +zstd < file -q --trace-file-stat -o file.zst +zstd -tq file.zst diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact new file mode 100644 index 00000000000..9183c467459 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact @@ -0,0 +1,24 @@ +Trace:FileStat: > UTIL_isConsole(0) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(/*stdin*\) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < -1 +Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file.zst) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file.zst) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_getFileSize(/*stdin*\) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < -1 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh new file mode 100755 index 00000000000..64f4b036216 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +datagen > file + +zstd < file -cq --trace-file-stat > file.zst +zstd -tq file.zst diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact new file mode 100644 index 00000000000..66b630e433c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact @@ -0,0 +1,18 @@ +Trace:FileStat: > UTIL_isConsole(0) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(1) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(/*stdin*\) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < -1 +Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) +Trace:FileStat: > UTIL_stat(-1, /*stdout*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_getFileSize(/*stdin*\) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < -1 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh new file mode 100755 index 00000000000..9e68f8f332a --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +datagen | zstd -q > file.zst +chmod 642 file.zst + +zstd -dq --trace-file-stat file.zst diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact new file mode 100644 index 00000000000..ad3a0deb868 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact @@ -0,0 +1,38 @@ +Trace:FileStat: > UTIL_isLink(file.zst) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(1) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isDirectory(file.zst) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isDirectoryStat() +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isSameFile(file.zst, file) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_setFileStat(4, file) +Trace:FileStat: > UTIL_stat(4, file) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_chmod(file, 0642) +Trace:FileStat: > fchmod +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_utime(file) +Trace:FileStat: < 0 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh new file mode 100755 index 00000000000..518c2a950d9 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +datagen | zstd -q > file.zst + +zstd -dcq --trace-file-stat file.zst > file diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact new file mode 100644 index 00000000000..8b60063c5fc --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact @@ -0,0 +1,18 @@ +Trace:FileStat: > UTIL_isLink(file.zst) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(1) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isDirectory(file.zst) +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isDirectoryStat() +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_stat(-1, file.zst) +Trace:FileStat: < 1 +Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) +Trace:FileStat: > UTIL_stat(-1, /*stdout*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh new file mode 100755 index 00000000000..135d75524d0 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +datagen | zstd -q > file.zst + +zstd -dcq --trace-file-stat < file.zst -o file diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact new file mode 100644 index 00000000000..716a7ac7b34 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact @@ -0,0 +1,20 @@ +Trace:FileStat: > UTIL_isConsole(0) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isDirectory(/*stdin*\) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(file) +Trace:FileStat: > UTIL_stat(-1, file) +Trace:FileStat: < 1 +Trace:FileStat: < 1 diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh new file mode 100755 index 00000000000..495f07b17fe --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +datagen | zstd -q > file.zst + +zstd -dcq --trace-file-stat < file.zst > file diff --git a/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact new file mode 100644 index 00000000000..2f6412059fc --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact @@ -0,0 +1,14 @@ +Trace:FileStat: > UTIL_isConsole(0) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(1) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isConsole(2) +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isDirectory(/*stdin*\) +Trace:FileStat: > UTIL_stat(-1, /*stdin*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 +Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) +Trace:FileStat: > UTIL_stat(-1, /*stdout*\) +Trace:FileStat: < 0 +Trace:FileStat: < 0 diff --git a/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh new file mode 100755 index 00000000000..708878f2643 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +#!/bin/sh + +. "$COMMON/platform.sh" + +set -e + +echo hello > hello +echo world > world + +zstd -q hello world + +println >&2 "Tests cases where progress information should not be printed" + +for args in \ + "" \ + "--fake-stderr-is-console -q" \ + "--fake-stderr-is-console -qq --progress" \ + "--no-progress --fake-stderr-is-console" \ + "--no-progress --fake-stderr-is-console -v" +do + println >&2 "args = $args" + println >&2 "compress file to file" + zstd $args -f hello + println >&2 "compress pipe to pipe" + zstd $args < hello > $INTOVOID + println >&2 "compress pipe to file" + zstd $args < hello -fo hello.zst + println >&2 "compress file to pipe" + zstd $args hello -c > $INTOVOID + println >&2 "compress 2 files" + zstd $args -f hello world + + println >&2 "decompress file to file" + zstd $args -d -f hello.zst + println >&2 "decompress pipe to pipe" + zstd $args -d < hello.zst > $INTOVOID + println >&2 "decompress pipe to file" + zstd $args -d < hello.zst -fo hello + println >&2 "decompress file to pipe" + zstd $args -d hello.zst -c > $INTOVOID + println >&2 "decompress 2 files" + zstd $args -d -f hello.zst world.zst + println >&2 "" +done diff --git a/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob new file mode 100644 index 00000000000..d0f91122769 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob @@ -0,0 +1,96 @@ +Tests cases where progress information should not be printed +args = +compress file to file +hello*hello.zst* +compress pipe to pipe +compress pipe to file +*stdin*hello.zst* +compress file to pipe +compress 2 files +2 files compressed* +decompress file to file +hello.zst* +decompress pipe to pipe +decompress pipe to file +*stdin* +decompress file to pipe +decompress 2 files +2 files decompressed* + +args = --fake-stderr-is-console -q +compress file to file +compress pipe to pipe +compress pipe to file +compress file to pipe +compress 2 files +decompress file to file +decompress pipe to pipe +decompress pipe to file +decompress file to pipe +decompress 2 files + +args = --fake-stderr-is-console -qq --progress +compress file to file +compress pipe to pipe +compress pipe to file +compress file to pipe +compress 2 files +decompress file to file +decompress pipe to pipe +decompress pipe to file +decompress file to pipe +decompress 2 files + +args = --no-progress --fake-stderr-is-console +compress file to file +hello*hello.zst* +compress pipe to pipe +compress pipe to file +*stdin*hello.zst* +compress file to pipe +compress 2 files +2 files compressed* +decompress file to file +hello.zst* +decompress pipe to pipe +decompress pipe to file +*stdin* +decompress file to pipe +decompress 2 files +2 files decompressed* + +args = --no-progress --fake-stderr-is-console -v +compress file to file +*Zstandard CLI* +hello*hello.zst* +compress pipe to pipe +*Zstandard CLI* +*stdin*stdout* +compress pipe to file +*Zstandard CLI* +*stdin*hello.zst* +compress file to pipe +*Zstandard CLI* +*hello*stdout* +compress 2 files +*Zstandard CLI* +*hello*hello.zst* +*world*world.zst* +2 files compressed* +decompress file to file +*Zstandard CLI* +hello.zst* +decompress pipe to pipe +*Zstandard CLI* +*stdin* +decompress pipe to file +*Zstandard CLI* +*stdin* +decompress file to pipe +*Zstandard CLI* +hello.zst* +decompress 2 files +*Zstandard CLI* +hello.zst* +world.zst* +2 files decompressed* diff --git a/3rdparty/zstd/tests/cli-tests/progress/progress.sh b/3rdparty/zstd/tests/cli-tests/progress/progress.sh new file mode 100755 index 00000000000..eb464993a24 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/progress/progress.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +. "$COMMON/platform.sh" + +set -e + +println >&2 "Tests cases where progress information should be printed" + +echo hello > hello +echo world > world + +zstd -q hello world + +for args in \ + "--progress" \ + "--fake-stderr-is-console" \ + "--progress --fake-stderr-is-console -q"; do + println >&2 "args = $args" + println >&2 "compress file to file" + zstd $args -f hello + println >&2 "compress pipe to pipe" + zstd $args < hello > $INTOVOID + println >&2 "compress pipe to file" + zstd $args < hello -fo hello.zst + println >&2 "compress file to pipe" + zstd $args hello -c > $INTOVOID + println >&2 "compress 2 files" + zstd $args -f hello world + + println >&2 "decompress file to file" + zstd $args -d -f hello.zst + println >&2 "decompress pipe to pipe" + zstd $args -d < hello.zst > $INTOVOID + println >&2 "decompress pipe to file" + zstd $args -d < hello.zst -fo hello + println >&2 "decompress file to pipe" + zstd $args -d hello.zst -c > $INTOVOID + println >&2 "decompress 2 files" + zstd $args -d -f hello.zst world.zst + println >&2 "" +done diff --git a/3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob b/3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob new file mode 100644 index 00000000000..ca620d3c282 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/progress/progress.sh.stderr.glob @@ -0,0 +1,62 @@ +Tests cases where progress information should be printed +args = --progress +compress file to file +*Read:*hello*hello.zst* +compress pipe to pipe +*Read:*stdin*stdout* +compress pipe to file +*Read:*stdin*hello.zst* +compress file to pipe +*Read:*hello*stdout* +compress 2 files +*Read*2 files compressed* +decompress file to file +*hello.zst*hello.zst* +decompress pipe to pipe +*stdin*stdin* +decompress pipe to file +*stdin*stdin* +decompress file to pipe +*hello.zst*hello.zst* +decompress 2 files +*hello.zst*2 files decompressed* + +args = --fake-stderr-is-console +compress file to file +*Read:*hello*hello.zst* +compress pipe to pipe +compress pipe to file +*Read:*stdin*hello.zst* +compress file to pipe +compress 2 files +*Read*2 files compressed* +decompress file to file +*hello.zst*hello.zst* +decompress pipe to pipe +decompress pipe to file +*stdin*stdin* +decompress file to pipe +decompress 2 files +*hello.zst*2 files decompressed* + +args = --progress --fake-stderr-is-console -q +compress file to file +*Read:*hello*hello.zst* +compress pipe to pipe +*Read:*stdin*stdout* +compress pipe to file +*Read:*stdin*hello.zst* +compress file to pipe +*Read:*hello*stdout* +compress 2 files +*Read*2 files compressed* +decompress file to file +*hello.zst*hello.zst* +decompress pipe to pipe +*stdin*stdin* +decompress pipe to file +*stdin*stdin* +decompress file to pipe +*hello.zst*hello.zst* +decompress 2 files +*hello.zst*2 files decompressed* diff --git a/3rdparty/zstd/tests/cli-tests/run.py b/3rdparty/zstd/tests/cli-tests/run.py new file mode 100755 index 00000000000..46564d2ffcb --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/run.py @@ -0,0 +1,731 @@ +#!/usr/bin/env python3 +# ################################################################ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ########################################################################## + +import argparse +import contextlib +import copy +import fnmatch +import os +import shutil +import subprocess +import sys +import tempfile +import typing + + +ZSTD_SYMLINKS = [ + "zstd", + "zstdmt", + "unzstd", + "zstdcat", + "zcat", + "gzip", + "gunzip", + "gzcat", + "lzma", + "unlzma", + "xz", + "unxz", + "lz4", + "unlz4", +] + + +EXCLUDED_DIRS = { + "bin", + "common", + "scratch", +} + + +EXCLUDED_BASENAMES = { + "setup", + "setup_once", + "teardown", + "teardown_once", + "README.md", + "run.py", + ".gitignore", +} + +EXCLUDED_SUFFIXES = [ + ".exact", + ".glob", + ".ignore", + ".exit", +] + + +def exclude_dir(dirname: str) -> bool: + """ + Should files under the directory :dirname: be excluded from the test runner? + """ + if dirname in EXCLUDED_DIRS: + return True + return False + + +def exclude_file(filename: str) -> bool: + """Should the file :filename: be excluded from the test runner?""" + if filename in EXCLUDED_BASENAMES: + return True + for suffix in EXCLUDED_SUFFIXES: + if filename.endswith(suffix): + return True + return False + +def read_file(filename: str) -> bytes: + """Reads the file :filename: and returns the contents as bytes.""" + with open(filename, "rb") as f: + return f.read() + + +def diff(a: bytes, b: bytes) -> str: + """Returns a diff between two different byte-strings :a: and :b:.""" + assert a != b + with tempfile.NamedTemporaryFile("wb") as fa: + fa.write(a) + fa.flush() + with tempfile.NamedTemporaryFile("wb") as fb: + fb.write(b) + fb.flush() + + diff_bytes = subprocess.run(["diff", fa.name, fb.name], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout + return diff_bytes.decode("utf8") + + +def pop_line(data: bytes) -> typing.Tuple[typing.Optional[bytes], bytes]: + """ + Pop the first line from :data: and returns the first line and the remainder + of the data as a tuple. If :data: is empty, returns :(None, data):. Otherwise + the first line always ends in a :\n:, even if it is the last line and :data: + doesn't end in :\n:. + """ + NEWLINE = b"\n" + + if data == b'': + return (None, data) + + parts = data.split(NEWLINE, maxsplit=1) + line = parts[0] + NEWLINE + if len(parts) == 1: + return line, b'' + + return line, parts[1] + + +def glob_line_matches(actual: bytes, expect: bytes) -> bool: + """ + Does the `actual` line match the expected glob line `expect`? + """ + return fnmatch.fnmatchcase(actual.strip(), expect.strip()) + + +def glob_diff(actual: bytes, expect: bytes) -> bytes: + """ + Returns None if the :actual: content matches the expected glob :expect:, + otherwise returns the diff bytes. + """ + diff = b'' + actual_line, actual = pop_line(actual) + expect_line, expect = pop_line(expect) + while True: + # Handle end of file conditions - allow extra newlines + while expect_line is None and actual_line == b"\n": + actual_line, actual = pop_line(actual) + while actual_line is None and expect_line == b"\n": + expect_line, expect = pop_line(expect) + + if expect_line is None and actual_line is None: + if diff == b'': + return None + return diff + elif expect_line is None: + diff += b"---\n" + while actual_line != None: + diff += b"> " + diff += actual_line + actual_line, actual = pop_line(actual) + return diff + elif actual_line is None: + diff += b"---\n" + while expect_line != None: + diff += b"< " + diff += expect_line + expect_line, expect = pop_line(expect) + return diff + + assert expect_line is not None + assert actual_line is not None + + if expect_line == b'...\n': + next_expect_line, expect = pop_line(expect) + if next_expect_line is None: + if diff == b'': + return None + return diff + while not glob_line_matches(actual_line, next_expect_line): + actual_line, actual = pop_line(actual) + if actual_line is None: + diff += b"---\n" + diff += b"< " + diff += next_expect_line + return diff + expect_line = next_expect_line + continue + + if not glob_line_matches(actual_line, expect_line): + diff += b'---\n' + diff += b'< ' + expect_line + diff += b'> ' + actual_line + + actual_line, actual = pop_line(actual) + expect_line, expect = pop_line(expect) + + +class Options: + """Options configuring how to run a :TestCase:.""" + def __init__( + self, + env: typing.Dict[str, str], + timeout: typing.Optional[int], + verbose: bool, + preserve: bool, + scratch_dir: str, + test_dir: str, + set_exact_output: bool, + ) -> None: + self.env = env + self.timeout = timeout + self.verbose = verbose + self.preserve = preserve + self.scratch_dir = scratch_dir + self.test_dir = test_dir + self.set_exact_output = set_exact_output + + +class TestCase: + """ + Logic and state related to running a single test case. + + 1. Initialize the test case. + 2. Launch the test case with :TestCase.launch():. + This will start the test execution in a subprocess, but + not wait for completion. So you could launch multiple test + cases in parallel. This will now print any test output. + 3. Analyze the results with :TestCase.analyze():. This will + join the test subprocess, check the results against the + expectations, and print the results to stdout. + + :TestCase.run(): is also provided which combines the launch & analyze + steps for single-threaded use-cases. + + All other methods, prefixed with _, are private helper functions. + """ + def __init__(self, test_filename: str, options: Options) -> None: + """ + Initialize the :TestCase: for the test located in :test_filename: + with the given :options:. + """ + self._opts = options + self._test_file = test_filename + self._test_name = os.path.normpath( + os.path.relpath(test_filename, start=self._opts.test_dir) + ) + self._success = {} + self._message = {} + self._test_stdin = None + self._scratch_dir = os.path.abspath(os.path.join(self._opts.scratch_dir, self._test_name)) + + @property + def name(self) -> str: + """Returns the unique name for the test.""" + return self._test_name + + def launch(self) -> None: + """ + Launch the test case as a subprocess, but do not block on completion. + This allows users to run multiple tests in parallel. Results aren't yet + printed out. + """ + self._launch_test() + + def analyze(self) -> bool: + """ + Must be called after :TestCase.launch():. Joins the test subprocess and + checks the results against expectations. Finally prints the results to + stdout and returns the success. + """ + self._join_test() + self._check_exit() + self._check_stderr() + self._check_stdout() + self._analyze_results() + return self._succeeded + + def run(self) -> bool: + """Shorthand for combining both :TestCase.launch(): and :TestCase.analyze():.""" + self.launch() + return self.analyze() + + def _log(self, *args, **kwargs) -> None: + """Logs test output.""" + print(file=sys.stdout, *args, **kwargs) + + def _vlog(self, *args, **kwargs) -> None: + """Logs verbose test output.""" + if self._opts.verbose: + print(file=sys.stdout, *args, **kwargs) + + def _test_environment(self) -> typing.Dict[str, str]: + """ + Returns the environment to be used for the + test subprocess. + """ + # We want to omit ZSTD cli flags so tests will be consistent across environments + env = {k: v for k, v in os.environ.items() if not k.startswith("ZSTD")} + for k, v in self._opts.env.items(): + self._vlog(f"${k}='{v}'") + env[k] = v + return env + + def _launch_test(self) -> None: + """Launch the test subprocess, but do not join it.""" + args = [os.path.abspath(self._test_file)] + stdin_name = f"{self._test_file}.stdin" + if os.path.exists(stdin_name): + self._test_stdin = open(stdin_name, "rb") + stdin = self._test_stdin + else: + stdin = subprocess.DEVNULL + cwd = self._scratch_dir + env = self._test_environment() + self._test_process = subprocess.Popen( + args=args, + stdin=stdin, + cwd=cwd, + env=env, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + def _join_test(self) -> None: + """Join the test process and save stderr, stdout, and the exit code.""" + (stdout, stderr) = self._test_process.communicate(timeout=self._opts.timeout) + self._output = {} + self._output["stdout"] = stdout + self._output["stderr"] = stderr + self._exit_code = self._test_process.returncode + self._test_process = None + if self._test_stdin is not None: + self._test_stdin.close() + self._test_stdin = None + + def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None: + """ + Check the output named :out_name: for an exact match against the :expected: content. + Saves the success and message. + """ + check_name = f"check_{out_name}" + actual = self._output[out_name] + if actual == expected: + self._success[check_name] = True + self._message[check_name] = f"{out_name} matches!" + else: + self._success[check_name] = False + self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}" + + if self._opts.set_exact_output: + with open(exact_name, "wb") as f: + f.write(actual) + + def _check_output_glob(self, out_name: str, expected: bytes) -> None: + """ + Check the output named :out_name: for a glob match against the :expected: glob. + Saves the success and message. + """ + check_name = f"check_{out_name}" + actual = self._output[out_name] + diff = glob_diff(actual, expected) + if diff is None: + self._success[check_name] = True + self._message[check_name] = f"{out_name} matches!" + else: + utf8_diff = diff.decode('utf8') + self._success[check_name] = False + self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{utf8_diff}" + + def _check_output(self, out_name: str) -> None: + """ + Checks the output named :out_name: for a match against the expectation. + We check for a .exact, .glob, and a .ignore file. If none are found we + expect that the output should be empty. + + If :Options.preserve: was set then we save the scratch directory and + save the stderr, stdout, and exit code to the scratch directory for + debugging. + """ + if self._opts.preserve: + # Save the output to the scratch directory + actual_name = os.path.join(self._scratch_dir, f"{out_name}") + with open(actual_name, "wb") as f: + f.write(self._output[out_name]) + + exact_name = f"{self._test_file}.{out_name}.exact" + glob_name = f"{self._test_file}.{out_name}.glob" + ignore_name = f"{self._test_file}.{out_name}.ignore" + + if os.path.exists(exact_name): + return self._check_output_exact(out_name, read_file(exact_name), exact_name) + elif os.path.exists(glob_name): + return self._check_output_glob(out_name, read_file(glob_name)) + else: + check_name = f"check_{out_name}" + self._success[check_name] = True + self._message[check_name] = f"{out_name} ignored!" + + def _check_stderr(self) -> None: + """Checks the stderr output against the expectation.""" + self._check_output("stderr") + + def _check_stdout(self) -> None: + """Checks the stdout output against the expectation.""" + self._check_output("stdout") + + def _check_exit(self) -> None: + """ + Checks the exit code against expectations. If a .exit file + exists, we expect that the exit code matches the contents. + Otherwise we expect the exit code to be zero. + + If :Options.preserve: is set we save the exit code to the + scratch directory under the filename "exit". + """ + if self._opts.preserve: + exit_name = os.path.join(self._scratch_dir, "exit") + with open(exit_name, "w") as f: + f.write(str(self._exit_code) + "\n") + exit_name = f"{self._test_file}.exit" + if os.path.exists(exit_name): + exit_code: int = int(read_file(exit_name)) + else: + exit_code: int = 0 + if exit_code == self._exit_code: + self._success["check_exit"] = True + self._message["check_exit"] = "Exit code matches!" + else: + self._success["check_exit"] = False + self._message["check_exit"] = f"Exit code mismatch! Expected {exit_code} but got {self._exit_code}" + + def _analyze_results(self) -> None: + """ + After all tests have been checked, collect all the successes + and messages, and print the results to stdout. + """ + STATUS = {True: "PASS", False: "FAIL"} + checks = sorted(self._success.keys()) + self._succeeded = all(self._success.values()) + self._log(f"{STATUS[self._succeeded]}: {self._test_name}") + + if not self._succeeded or self._opts.verbose: + for check in checks: + if self._opts.verbose or not self._success[check]: + self._log(f"{STATUS[self._success[check]]}: {self._test_name}.{check}") + self._log(self._message[check]) + + self._log("----------------------------------------") + + +class TestSuite: + """ + Setup & teardown test suite & cases. + This class is intended to be used as a context manager. + + TODO: Make setup/teardown failure emit messages, not throw exceptions. + """ + def __init__(self, test_directory: str, options: Options) -> None: + self._opts = options + self._test_dir = os.path.abspath(test_directory) + rel_test_dir = os.path.relpath(test_directory, start=self._opts.test_dir) + assert not rel_test_dir.startswith(os.path.sep) + self._scratch_dir = os.path.normpath(os.path.join(self._opts.scratch_dir, rel_test_dir)) + + def __enter__(self) -> 'TestSuite': + self._setup_once() + return self + + def __exit__(self, _exc_type, _exc_value, _traceback) -> None: + self._teardown_once() + + @contextlib.contextmanager + def test_case(self, test_basename: str) -> TestCase: + """ + Context manager for a test case in the test suite. + Pass the basename of the test relative to the :test_directory:. + """ + assert os.path.dirname(test_basename) == "" + try: + self._setup(test_basename) + test_filename = os.path.join(self._test_dir, test_basename) + yield TestCase(test_filename, self._opts) + finally: + self._teardown(test_basename) + + def _remove_scratch_dir(self, dir: str) -> None: + """Helper to remove a scratch directory with sanity checks""" + assert "scratch" in dir + assert dir.startswith(self._scratch_dir) + assert os.path.exists(dir) + shutil.rmtree(dir) + + def _setup_once(self) -> None: + if os.path.exists(self._scratch_dir): + self._remove_scratch_dir(self._scratch_dir) + os.makedirs(self._scratch_dir) + setup_script = os.path.join(self._test_dir, "setup_once") + if os.path.exists(setup_script): + self._run_script(setup_script, cwd=self._scratch_dir) + + def _teardown_once(self) -> None: + assert os.path.exists(self._scratch_dir) + teardown_script = os.path.join(self._test_dir, "teardown_once") + if os.path.exists(teardown_script): + self._run_script(teardown_script, cwd=self._scratch_dir) + if not self._opts.preserve: + self._remove_scratch_dir(self._scratch_dir) + + def _setup(self, test_basename: str) -> None: + test_scratch_dir = os.path.join(self._scratch_dir, test_basename) + assert not os.path.exists(test_scratch_dir) + os.makedirs(test_scratch_dir) + setup_script = os.path.join(self._test_dir, "setup") + if os.path.exists(setup_script): + self._run_script(setup_script, cwd=test_scratch_dir) + + def _teardown(self, test_basename: str) -> None: + test_scratch_dir = os.path.join(self._scratch_dir, test_basename) + assert os.path.exists(test_scratch_dir) + teardown_script = os.path.join(self._test_dir, "teardown") + if os.path.exists(teardown_script): + self._run_script(teardown_script, cwd=test_scratch_dir) + if not self._opts.preserve: + self._remove_scratch_dir(test_scratch_dir) + + def _run_script(self, script: str, cwd: str) -> None: + env = copy.copy(os.environ) + for k, v in self._opts.env.items(): + env[k] = v + try: + subprocess.run( + args=[script], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + env=env, + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"{script} failed with exit code {e.returncode}!") + print(f"stderr:\n{e.stderr}") + print(f"stdout:\n{e.stdout}") + raise + +TestSuites = typing.Dict[str, typing.List[str]] + +def get_all_tests(options: Options) -> TestSuites: + """ + Find all the test in the test directory and return the test suites. + """ + test_suites = {} + for root, dirs, files in os.walk(options.test_dir, topdown=True): + dirs[:] = [d for d in dirs if not exclude_dir(d)] + test_cases = [] + for file in files: + if not exclude_file(file): + test_cases.append(file) + assert root == os.path.normpath(root) + test_suites[root] = test_cases + return test_suites + + +def resolve_listed_tests( + tests: typing.List[str], options: Options +) -> TestSuites: + """ + Resolve the list of tests passed on the command line into their + respective test suites. Tests can either be paths, or test names + relative to the test directory. + """ + test_suites = {} + for test in tests: + if not os.path.exists(test): + test = os.path.join(options.test_dir, test) + if not os.path.exists(test): + raise RuntimeError(f"Test {test} does not exist!") + + test = os.path.normpath(os.path.abspath(test)) + assert test.startswith(options.test_dir) + test_suite = os.path.dirname(test) + test_case = os.path.basename(test) + test_suites.setdefault(test_suite, []).append(test_case) + + return test_suites + +def run_tests(test_suites: TestSuites, options: Options) -> bool: + """ + Runs all the test in the :test_suites: with the given :options:. + Prints the results to stdout. + """ + tests = {} + for test_dir, test_files in test_suites.items(): + with TestSuite(test_dir, options) as test_suite: + test_files = sorted(set(test_files)) + for test_file in test_files: + with test_suite.test_case(test_file) as test_case: + tests[test_case.name] = test_case.run() + + successes = 0 + for test, status in tests.items(): + if status: + successes += 1 + else: + print(f"FAIL: {test}") + if successes == len(tests): + print(f"PASSED all {len(tests)} tests!") + return True + else: + print(f"FAILED {len(tests) - successes} / {len(tests)} tests!") + return False + + +def setup_zstd_symlink_dir(zstd_symlink_dir: str, zstd: str) -> None: + assert os.path.join("bin", "symlinks") in zstd_symlink_dir + if not os.path.exists(zstd_symlink_dir): + os.makedirs(zstd_symlink_dir) + for symlink in ZSTD_SYMLINKS: + path = os.path.join(zstd_symlink_dir, symlink) + if os.path.exists(path): + os.remove(path) + os.symlink(zstd, path) + +if __name__ == "__main__": + CLI_TEST_DIR = os.path.dirname(sys.argv[0]) + REPO_DIR = os.path.join(CLI_TEST_DIR, "..", "..") + PROGRAMS_DIR = os.path.join(REPO_DIR, "programs") + TESTS_DIR = os.path.join(REPO_DIR, "tests") + ZSTD_PATH = os.path.join(PROGRAMS_DIR, "zstd") + ZSTDGREP_PATH = os.path.join(PROGRAMS_DIR, "zstdgrep") + ZSTDLESS_PATH = os.path.join(PROGRAMS_DIR, "zstdless") + DATAGEN_PATH = os.path.join(TESTS_DIR, "datagen") + + parser = argparse.ArgumentParser( + ( + "Runs the zstd CLI tests. Exits nonzero on failure. Default arguments are\n" + "generally correct. Pass --preserve to preserve test output for debugging,\n" + "and --verbose to get verbose test output.\n" + ) + ) + parser.add_argument( + "--preserve", + action="store_true", + help="Preserve the scratch directory TEST_DIR/scratch/ for debugging purposes." + ) + parser.add_argument("--verbose", action="store_true", help="Verbose test output.") + parser.add_argument("--timeout", default=200, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.") + parser.add_argument( + "--exec-prefix", + default=None, + help="Sets the EXEC_PREFIX environment variable. Prefix to invocations of the zstd CLI." + ) + parser.add_argument( + "--zstd", + default=ZSTD_PATH, + help="Sets the ZSTD_BIN environment variable. Path of the zstd CLI." + ) + parser.add_argument( + "--zstdgrep", + default=ZSTDGREP_PATH, + help="Sets the ZSTDGREP_BIN environment variable. Path of the zstdgrep CLI." + ) + parser.add_argument( + "--zstdless", + default=ZSTDLESS_PATH, + help="Sets the ZSTDLESS_BIN environment variable. Path of the zstdless CLI." + ) + parser.add_argument( + "--datagen", + default=DATAGEN_PATH, + help="Sets the DATAGEN_BIN environment variable. Path to the datagen CLI." + ) + parser.add_argument( + "--test-dir", + default=CLI_TEST_DIR, + help=( + "Runs the tests under this directory. " + "Adds TEST_DIR/bin/ to path. " + "Scratch directory located in TEST_DIR/scratch/." + ) + ) + parser.add_argument( + "--set-exact-output", + action="store_true", + help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists" + ) + parser.add_argument( + "tests", + nargs="*", + help="Run only these test cases. Can either be paths or test names relative to TEST_DIR/" + ) + args = parser.parse_args() + + if args.timeout <= 0: + args.timeout = None + + args.test_dir = os.path.normpath(os.path.abspath(args.test_dir)) + bin_dir = os.path.abspath(os.path.join(args.test_dir, "bin")) + zstd_symlink_dir = os.path.join(bin_dir, "symlinks") + scratch_dir = os.path.join(args.test_dir, "scratch") + + setup_zstd_symlink_dir(zstd_symlink_dir, os.path.abspath(args.zstd)) + + env = {} + if args.exec_prefix is not None: + env["EXEC_PREFIX"] = args.exec_prefix + env["ZSTD_SYMLINK_DIR"] = zstd_symlink_dir + env["ZSTD_REPO_DIR"] = os.path.abspath(REPO_DIR) + env["DATAGEN_BIN"] = os.path.abspath(args.datagen) + env["ZSTDGREP_BIN"] = os.path.abspath(args.zstdgrep) + env["ZSTDLESS_BIN"] = os.path.abspath(args.zstdless) + env["COMMON"] = os.path.abspath(os.path.join(args.test_dir, "common")) + env["PATH"] = bin_dir + ":" + os.getenv("PATH", "") + env["LC_ALL"] = "C" + + opts = Options( + env=env, + timeout=args.timeout, + verbose=args.verbose, + preserve=args.preserve, + test_dir=args.test_dir, + scratch_dir=scratch_dir, + set_exact_output=args.set_exact_output, + ) + + if len(args.tests) == 0: + tests = get_all_tests(opts) + else: + tests = resolve_listed_tests(args.tests, opts) + + success = run_tests(tests, opts) + if success: + sys.exit(0) + else: + sys.exit(1) diff --git a/3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup new file mode 100755 index 00000000000..cf391ed2117 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/setup @@ -0,0 +1,6 @@ +#!/bin/sh +set -e + +println "hello" > hello +println "world" > world +zstd hello world diff --git a/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh new file mode 100755 index 00000000000..74ec063d13c --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +# Test zstdcat symlink in bin/ +zstdcat hello.zst +zstdcat hello.zst world +zstdcat hello world.zst +zstdcat hello.zst world.zst + +# Test local zstdcat symlink +ln -s $(which zstd) ./zstdcat +./zstdcat hello.zst diff --git a/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact new file mode 100644 index 00000000000..3205b059b34 --- /dev/null +++ b/3rdparty/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact @@ -0,0 +1,8 @@ +hello +hello +world +hello +world +hello +world +hello |