diff options
Diffstat (limited to '3rdparty/zstd/contrib/externalSequenceProducer')
6 files changed, 269 insertions, 0 deletions
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/.gitignore b/3rdparty/zstd/contrib/externalSequenceProducer/.gitignore new file mode 100644 index 00000000000..147710aee78 --- /dev/null +++ b/3rdparty/zstd/contrib/externalSequenceProducer/.gitignore @@ -0,0 +1,2 @@ +# build artifacts +externalSequenceProducer diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/Makefile b/3rdparty/zstd/contrib/externalSequenceProducer/Makefile new file mode 100644 index 00000000000..0591ae01ba5 --- /dev/null +++ b/3rdparty/zstd/contrib/externalSequenceProducer/Makefile @@ -0,0 +1,40 @@ +# ################################################################ +# Copyright (c) Yann Collet, Meta Platforms, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +PROGDIR = ../../programs +LIBDIR = ../../lib + +LIBZSTD = $(LIBDIR)/libzstd.a + +CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common + +CFLAGS ?= -O3 +CFLAGS += -std=gnu99 +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum \ + -Wstrict-prototypes -Wundef -Wpointer-arith \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + +default: externalSequenceProducer + +all: externalSequenceProducer + +externalSequenceProducer: sequence_producer.c main.c $(LIBZSTD) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +.PHONY: $(LIBZSTD) +$(LIBZSTD): + $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)" + +clean: + $(RM) *.o + $(MAKE) -C $(LIBDIR) clean > /dev/null + $(RM) externalSequenceProducer diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/README.md b/3rdparty/zstd/contrib/externalSequenceProducer/README.md new file mode 100644 index 00000000000..c16a170073b --- /dev/null +++ b/3rdparty/zstd/contrib/externalSequenceProducer/README.md @@ -0,0 +1,14 @@ +externalSequenceProducer +===================== + +`externalSequenceProducer` is a test tool for the Block-Level Sequence Producer API. +It demonstrates how to use the API to perform a simple round-trip test. + +A sample sequence producer is provided in sequence_producer.c, but the user can swap +this out with a different one if desired. The sample sequence producer implements +LZ parsing with a 1KB hashtable. Dictionary-based parsing is not currently supported. + +Command line : +``` +externalSequenceProducer filename +``` diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/main.c b/3rdparty/zstd/contrib/externalSequenceProducer/main.c new file mode 100644 index 00000000000..e67e295383c --- /dev/null +++ b/3rdparty/zstd/contrib/externalSequenceProducer/main.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "zstd_errors.h" +#include "sequence_producer.h" // simpleSequenceProducer + +#define CHECK(res) \ +do { \ + if (ZSTD_isError(res)) { \ + printf("ERROR: %s\n", ZSTD_getErrorName(res)); \ + return 1; \ + } \ +} while (0) \ + +int main(int argc, char *argv[]) { + if (argc != 2) { + printf("Usage: externalSequenceProducer <file>\n"); + return 1; + } + + ZSTD_CCtx* const zc = ZSTD_createCCtx(); + + int simpleSequenceProducerState = 0xdeadbeef; + + // Here is the crucial bit of code! + ZSTD_registerSequenceProducer( + zc, + &simpleSequenceProducerState, + simpleSequenceProducer + ); + + { + size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 1); + CHECK(res); + } + + FILE *f = fopen(argv[1], "rb"); + assert(f); + { + int const ret = fseek(f, 0, SEEK_END); + assert(ret == 0); + } + size_t const srcSize = ftell(f); + { + int const ret = fseek(f, 0, SEEK_SET); + assert(ret == 0); + } + + char* const src = malloc(srcSize + 1); + assert(src); + { + size_t const ret = fread(src, srcSize, 1, f); + assert(ret == 1); + int const ret2 = fclose(f); + assert(ret2 == 0); + } + + size_t const dstSize = ZSTD_compressBound(srcSize); + char* const dst = malloc(dstSize); + assert(dst); + + size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize); + CHECK(cSize); + + char* const val = malloc(srcSize); + assert(val); + + { + size_t const res = ZSTD_decompress(val, srcSize, dst, cSize); + CHECK(res); + } + + if (memcmp(src, val, srcSize) == 0) { + printf("Compression and decompression were successful!\n"); + printf("Original size: %lu\n", srcSize); + printf("Compressed size: %lu\n", cSize); + } else { + printf("ERROR: input and validation buffers don't match!\n"); + for (size_t i = 0; i < srcSize; i++) { + if (src[i] != val[i]) { + printf("First bad index: %zu\n", i); + break; + } + } + return 1; + } + + ZSTD_freeCCtx(zc); + free(src); + free(dst); + free(val); + return 0; +} diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c new file mode 100644 index 00000000000..60a2f9572b0 --- /dev/null +++ b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "sequence_producer.h" + +#define HSIZE 1024 +static U32 const HLOG = 10; +static U32 const MLS = 4; +static U32 const BADIDX = 0xffffffff; + +size_t simpleSequenceProducer( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +) { + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + const BYTE* anchor = istart; + size_t seqCount = 0; + U32 hashTable[HSIZE]; + + (void)sequenceProducerState; + (void)dict; + (void)dictSize; + (void)outSeqsCapacity; + (void)compressionLevel; + + { int i; + for (i=0; i < HSIZE; i++) { + hashTable[i] = BADIDX; + } } + + while (ip + MLS < iend) { + size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS); + U32 const matchIndex = hashTable[hash]; + hashTable[hash] = (U32)(ip - istart); + + if (matchIndex != BADIDX) { + const BYTE* const match = istart + matchIndex; + U32 const matchLen = (U32)ZSTD_count(ip, match, iend); + if (matchLen >= ZSTD_MINMATCH_MIN) { + U32 const litLen = (U32)(ip - anchor); + U32 const offset = (U32)(ip - match); + ZSTD_Sequence const seq = { + offset, litLen, matchLen, 0 + }; + + /* Note: it's crucial to stay within the window size! */ + if (offset <= windowSize) { + outSeqs[seqCount++] = seq; + ip += matchLen; + anchor = ip; + continue; + } + } + } + + ip++; + } + + { ZSTD_Sequence const finalSeq = { + 0, (U32)(iend - anchor), 0, 0 + }; + outSeqs[seqCount++] = finalSeq; + } + + return seqCount; +} diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h new file mode 100644 index 00000000000..19f9982ac36 --- /dev/null +++ b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MATCHFINDER_H +#define MATCHFINDER_H + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +size_t simpleSequenceProducer( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +#endif |