summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/zstd/contrib/externalSequenceProducer
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/zstd/contrib/externalSequenceProducer')
-rw-r--r--3rdparty/zstd/contrib/externalSequenceProducer/.gitignore2
-rw-r--r--3rdparty/zstd/contrib/externalSequenceProducer/Makefile40
-rw-r--r--3rdparty/zstd/contrib/externalSequenceProducer/README.md14
-rw-r--r--3rdparty/zstd/contrib/externalSequenceProducer/main.c107
-rw-r--r--3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c80
-rw-r--r--3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h26
6 files changed, 269 insertions, 0 deletions
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/.gitignore b/3rdparty/zstd/contrib/externalSequenceProducer/.gitignore
new file mode 100644
index 00000000000..147710aee78
--- /dev/null
+++ b/3rdparty/zstd/contrib/externalSequenceProducer/.gitignore
@@ -0,0 +1,2 @@
+# build artifacts
+externalSequenceProducer
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/Makefile b/3rdparty/zstd/contrib/externalSequenceProducer/Makefile
new file mode 100644
index 00000000000..0591ae01ba5
--- /dev/null
+++ b/3rdparty/zstd/contrib/externalSequenceProducer/Makefile
@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) Yann Collet, Meta Platforms, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common
+
+CFLAGS ?= -O3
+CFLAGS += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum \
+ -Wstrict-prototypes -Wundef -Wpointer-arith \
+ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+ -Wredundant-decls
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+
+default: externalSequenceProducer
+
+all: externalSequenceProducer
+
+externalSequenceProducer: sequence_producer.c main.c $(LIBZSTD)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+ $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+clean:
+ $(RM) *.o
+ $(MAKE) -C $(LIBDIR) clean > /dev/null
+ $(RM) externalSequenceProducer
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/README.md b/3rdparty/zstd/contrib/externalSequenceProducer/README.md
new file mode 100644
index 00000000000..c16a170073b
--- /dev/null
+++ b/3rdparty/zstd/contrib/externalSequenceProducer/README.md
@@ -0,0 +1,14 @@
+externalSequenceProducer
+=====================
+
+`externalSequenceProducer` is a test tool for the Block-Level Sequence Producer API.
+It demonstrates how to use the API to perform a simple round-trip test.
+
+A sample sequence producer is provided in sequence_producer.c, but the user can swap
+this out with a different one if desired. The sample sequence producer implements
+LZ parsing with a 1KB hashtable. Dictionary-based parsing is not currently supported.
+
+Command line :
+```
+externalSequenceProducer filename
+```
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/main.c b/3rdparty/zstd/contrib/externalSequenceProducer/main.c
new file mode 100644
index 00000000000..e67e295383c
--- /dev/null
+++ b/3rdparty/zstd/contrib/externalSequenceProducer/main.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "zstd_errors.h"
+#include "sequence_producer.h" // simpleSequenceProducer
+
+#define CHECK(res) \
+do { \
+ if (ZSTD_isError(res)) { \
+ printf("ERROR: %s\n", ZSTD_getErrorName(res)); \
+ return 1; \
+ } \
+} while (0) \
+
+int main(int argc, char *argv[]) {
+ if (argc != 2) {
+ printf("Usage: externalSequenceProducer <file>\n");
+ return 1;
+ }
+
+ ZSTD_CCtx* const zc = ZSTD_createCCtx();
+
+ int simpleSequenceProducerState = 0xdeadbeef;
+
+ // Here is the crucial bit of code!
+ ZSTD_registerSequenceProducer(
+ zc,
+ &simpleSequenceProducerState,
+ simpleSequenceProducer
+ );
+
+ {
+ size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 1);
+ CHECK(res);
+ }
+
+ FILE *f = fopen(argv[1], "rb");
+ assert(f);
+ {
+ int const ret = fseek(f, 0, SEEK_END);
+ assert(ret == 0);
+ }
+ size_t const srcSize = ftell(f);
+ {
+ int const ret = fseek(f, 0, SEEK_SET);
+ assert(ret == 0);
+ }
+
+ char* const src = malloc(srcSize + 1);
+ assert(src);
+ {
+ size_t const ret = fread(src, srcSize, 1, f);
+ assert(ret == 1);
+ int const ret2 = fclose(f);
+ assert(ret2 == 0);
+ }
+
+ size_t const dstSize = ZSTD_compressBound(srcSize);
+ char* const dst = malloc(dstSize);
+ assert(dst);
+
+ size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize);
+ CHECK(cSize);
+
+ char* const val = malloc(srcSize);
+ assert(val);
+
+ {
+ size_t const res = ZSTD_decompress(val, srcSize, dst, cSize);
+ CHECK(res);
+ }
+
+ if (memcmp(src, val, srcSize) == 0) {
+ printf("Compression and decompression were successful!\n");
+ printf("Original size: %lu\n", srcSize);
+ printf("Compressed size: %lu\n", cSize);
+ } else {
+ printf("ERROR: input and validation buffers don't match!\n");
+ for (size_t i = 0; i < srcSize; i++) {
+ if (src[i] != val[i]) {
+ printf("First bad index: %zu\n", i);
+ break;
+ }
+ }
+ return 1;
+ }
+
+ ZSTD_freeCCtx(zc);
+ free(src);
+ free(dst);
+ free(val);
+ return 0;
+}
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c
new file mode 100644
index 00000000000..60a2f9572b0
--- /dev/null
+++ b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "zstd_compress_internal.h"
+#include "sequence_producer.h"
+
+#define HSIZE 1024
+static U32 const HLOG = 10;
+static U32 const MLS = 4;
+static U32 const BADIDX = 0xffffffff;
+
+size_t simpleSequenceProducer(
+ void* sequenceProducerState,
+ ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ int compressionLevel,
+ size_t windowSize
+) {
+ const BYTE* const istart = (const BYTE*)src;
+ const BYTE* const iend = istart + srcSize;
+ const BYTE* ip = istart;
+ const BYTE* anchor = istart;
+ size_t seqCount = 0;
+ U32 hashTable[HSIZE];
+
+ (void)sequenceProducerState;
+ (void)dict;
+ (void)dictSize;
+ (void)outSeqsCapacity;
+ (void)compressionLevel;
+
+ { int i;
+ for (i=0; i < HSIZE; i++) {
+ hashTable[i] = BADIDX;
+ } }
+
+ while (ip + MLS < iend) {
+ size_t const hash = ZSTD_hashPtr(ip, HLOG, MLS);
+ U32 const matchIndex = hashTable[hash];
+ hashTable[hash] = (U32)(ip - istart);
+
+ if (matchIndex != BADIDX) {
+ const BYTE* const match = istart + matchIndex;
+ U32 const matchLen = (U32)ZSTD_count(ip, match, iend);
+ if (matchLen >= ZSTD_MINMATCH_MIN) {
+ U32 const litLen = (U32)(ip - anchor);
+ U32 const offset = (U32)(ip - match);
+ ZSTD_Sequence const seq = {
+ offset, litLen, matchLen, 0
+ };
+
+ /* Note: it's crucial to stay within the window size! */
+ if (offset <= windowSize) {
+ outSeqs[seqCount++] = seq;
+ ip += matchLen;
+ anchor = ip;
+ continue;
+ }
+ }
+ }
+
+ ip++;
+ }
+
+ { ZSTD_Sequence const finalSeq = {
+ 0, (U32)(iend - anchor), 0, 0
+ };
+ outSeqs[seqCount++] = finalSeq;
+ }
+
+ return seqCount;
+}
diff --git a/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h
new file mode 100644
index 00000000000..19f9982ac36
--- /dev/null
+++ b/3rdparty/zstd/contrib/externalSequenceProducer/sequence_producer.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Yann Collet, Meta Platforms, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MATCHFINDER_H
+#define MATCHFINDER_H
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+size_t simpleSequenceProducer(
+ void* sequenceProducerState,
+ ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize,
+ int compressionLevel,
+ size_t windowSize
+);
+
+#endif