summaryrefslogtreecommitdiffstatshomepage
path: root/src/lib
diff options
context:
space:
mode:
author Vas Crabb <cuavas@users.noreply.github.com>2023-12-11 10:48:02 +1100
committer GitHub <noreply@github.com>2023-12-11 10:48:02 +1100
commit05e69b43e9770e16c4bf1c6ca3e585e64c3f6ff9 (patch)
tree00ff2118ddc01455ee55785283a2fcfb9bfaf77e /src/lib
parent1a4287925e3982e2f782dee6f62b198984b839de (diff)
Added Zstandard support for zip archives and CHDs. (#11827)
* 3rdparty/zstd: Added Zstandard compression library version 1.5.5. * util/unzip.cpp: Added support for Zstandard compression (method 93). * util/chdcodec.cpp: Added support for Zstandard compression. * 3rdparty/flac: Always define NDEBUG to avoid log spam.
Diffstat (limited to 'src/lib')
-rw-r--r--src/lib/util/chd.cpp14
-rw-r--r--src/lib/util/chdcodec.cpp168
-rw-r--r--src/lib/util/chdcodec.h2
-rw-r--r--src/lib/util/unzip.cpp95
4 files changed, 269 insertions, 10 deletions
diff --git a/src/lib/util/chd.cpp b/src/lib/util/chd.cpp
index 11f0b93eb86..194884bfd9a 100644
--- a/src/lib/util/chd.cpp
+++ b/src/lib/util/chd.cpp
@@ -106,12 +106,12 @@ enum
// description of where a metadata entry lives within the file
struct chd_file::metadata_entry
{
- uint64_t offset; // offset within the file of the header
- uint64_t next; // offset within the file of the next header
- uint64_t prev; // offset within the file of the previous header
- uint32_t length; // length of the metadata
- uint32_t metatag; // metadata tag
- uint8_t flags; // flag bits
+ uint64_t offset; // offset within the file of the header
+ uint64_t next; // offset within the file of the next header
+ uint64_t prev; // offset within the file of the previous header
+ uint32_t length; // length of the metadata
+ uint32_t metatag; // metadata tag
+ uint8_t flags; // flag bits
};
@@ -119,7 +119,7 @@ struct chd_file::metadata_entry
struct chd_file::metadata_hash
{
- uint8_t tag[4]; // tag of the metadata in big-endian
+ uint8_t tag[4]; // tag of the metadata in big-endian
util::sha1_t sha1; // hash data
};
diff --git a/src/lib/util/chdcodec.cpp b/src/lib/util/chdcodec.cpp
index fa46975285e..c59d880374c 100644
--- a/src/lib/util/chdcodec.cpp
+++ b/src/lib/util/chdcodec.cpp
@@ -21,6 +21,7 @@
#include "lzma/C/LzmaEnc.h"
#include <zlib.h>
+#include <zstd.h>
#include <new>
@@ -103,6 +104,44 @@ private:
};
+// ======================> chd_zstd_compressor
+
+// Zstandard compressor
+class chd_zstd_compressor : public chd_compressor
+{
+public:
+ // construction/destruction
+ chd_zstd_compressor(chd_file &chd, uint32_t hunkbytes, bool lossy);
+ ~chd_zstd_compressor();
+
+ // core functionality
+ virtual uint32_t compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) override;
+
+private:
+ // internal state
+ ZSTD_CStream * m_stream;
+};
+
+
+// ======================> chd_zstd_decompressor
+
+// Zstandard decompressor
+class chd_zstd_decompressor : public chd_decompressor
+{
+public:
+ // construction/destruction
+ chd_zstd_decompressor(chd_file &chd, uint32_t hunkbytes, bool lossy);
+ ~chd_zstd_decompressor();
+
+ // core functionality
+ virtual void decompress(const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) override;
+
+private:
+ // internal state
+ ZSTD_DStream * m_stream;
+};
+
+
// ======================> chd_lzma_allocator
// allocation helper clas for zlib
@@ -493,12 +532,14 @@ const codec_entry f_codec_list[] =
{
// general codecs
{ CHD_CODEC_ZLIB, false, "Deflate", &codec_entry::construct_compressor<chd_zlib_compressor>, &codec_entry::construct_decompressor<chd_zlib_decompressor> },
+ { CHD_CODEC_ZSTD, false, "Zstandard", &codec_entry::construct_compressor<chd_zstd_compressor>, &codec_entry::construct_decompressor<chd_zstd_decompressor> },
{ CHD_CODEC_LZMA, false, "LZMA", &codec_entry::construct_compressor<chd_lzma_compressor>, &codec_entry::construct_decompressor<chd_lzma_decompressor> },
{ CHD_CODEC_HUFFMAN, false, "Huffman", &codec_entry::construct_compressor<chd_huffman_compressor>, &codec_entry::construct_decompressor<chd_huffman_decompressor> },
{ CHD_CODEC_FLAC, false, "FLAC", &codec_entry::construct_compressor<chd_flac_compressor>, &codec_entry::construct_decompressor<chd_flac_decompressor> },
// general codecs with CD frontend
{ CHD_CODEC_CD_ZLIB, false, "CD Deflate", &codec_entry::construct_compressor<chd_cd_compressor<chd_zlib_compressor, chd_zlib_compressor> >, &codec_entry::construct_decompressor<chd_cd_decompressor<chd_zlib_decompressor, chd_zlib_decompressor> > },
+ { CHD_CODEC_CD_ZSTD, false, "CD Zstandard", &codec_entry::construct_compressor<chd_cd_compressor<chd_zstd_compressor, chd_zstd_compressor> >, &codec_entry::construct_decompressor<chd_cd_decompressor<chd_zstd_decompressor, chd_zstd_decompressor> > },
{ CHD_CODEC_CD_LZMA, false, "CD LZMA", &codec_entry::construct_compressor<chd_cd_compressor<chd_lzma_compressor, chd_zlib_compressor> >, &codec_entry::construct_decompressor<chd_cd_decompressor<chd_lzma_decompressor, chd_zlib_decompressor> > },
{ CHD_CODEC_CD_FLAC, false, "CD FLAC", &codec_entry::construct_compressor<chd_cd_flac_compressor>, &codec_entry::construct_decompressor<chd_cd_flac_decompressor> },
@@ -896,7 +937,7 @@ chd_zlib_compressor::~chd_zlib_compressor()
uint32_t chd_zlib_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest)
{
- // reset the decompressor
+ // reset the compressor
m_deflater.next_in = const_cast<Bytef *>(src);
m_deflater.avail_in = srclen;
m_deflater.total_in = 0;
@@ -984,6 +1025,131 @@ void chd_zlib_decompressor::decompress(const uint8_t *src, uint32_t complen, uin
//**************************************************************************
+// ZSTANDARD COMPRESSOR
+//**************************************************************************
+
+//-------------------------------------------------
+// chd_zstd_compressor - constructor
+//-------------------------------------------------
+
+chd_zstd_compressor::chd_zstd_compressor(chd_file &chd, uint32_t hunkbytes, bool lossy)
+ : chd_compressor(chd, hunkbytes, lossy)
+ , m_stream(nullptr)
+{
+ // initialize the stream
+ m_stream = ZSTD_createCStream();
+
+ // convert errors
+ if (!m_stream)
+ throw std::bad_alloc();
+}
+
+
+//-------------------------------------------------
+// ~chd_zstd_compressor - destructor
+//-------------------------------------------------
+
+chd_zstd_compressor::~chd_zstd_compressor()
+{
+ ZSTD_freeCStream(m_stream);
+}
+
+
+//-------------------------------------------------
+// compress - compress data using the Zstandard
+// codec
+//-------------------------------------------------
+
+uint32_t chd_zstd_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest)
+{
+ // reset the compressor
+ auto result = ZSTD_initCStream(m_stream, ZSTD_maxCLevel());
+ if (ZSTD_isError(result))
+ throw std::error_condition(chd_file::error::COMPRESSION_ERROR);
+
+ // do it
+ ZSTD_inBuffer input{ src, srclen, 0 };
+ ZSTD_outBuffer output = { dest, srclen, 0 };
+ while (output.pos < output.size)
+ {
+ result = ZSTD_compressStream2(m_stream, &output, &input, ZSTD_e_end);
+ if (ZSTD_isError(result))
+ throw std::error_condition(chd_file::error::COMPRESSION_ERROR);
+ else if (!result)
+ break;
+ }
+
+ // if we ended up with more data than we started with, return an error
+ if (output.pos == output.size)
+ throw std::error_condition(chd_file::error::COMPRESSION_ERROR);
+
+ // otherwise, return the length
+ return output.pos;
+}
+
+
+
+//**************************************************************************
+// ZSTANDARD DECOMPRESSOR
+//**************************************************************************
+
+//-------------------------------------------------
+// chd_zstd_compressor - constructor
+//-------------------------------------------------
+
+chd_zstd_decompressor::chd_zstd_decompressor(chd_file &chd, uint32_t hunkbytes, bool lossy)
+ : chd_decompressor(chd, hunkbytes, lossy)
+ , m_stream(nullptr)
+{
+ // initialize the stream
+ m_stream = ZSTD_createDStream();
+
+ // convert errors
+ if (!m_stream)
+ throw std::bad_alloc();
+}
+
+
+//-------------------------------------------------
+// ~chd_zstd_decompressor - destructor
+//-------------------------------------------------
+
+chd_zstd_decompressor::~chd_zstd_decompressor()
+{
+ ZSTD_freeDStream(m_stream);
+}
+
+
+//-------------------------------------------------
+// decompress - decompress data using the
+// Zstandard codec
+//-------------------------------------------------
+
+void chd_zstd_decompressor::decompress(const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen)
+{
+ // reset the decompressor
+ auto result = ZSTD_initDStream(m_stream);
+ if (ZSTD_isError(result))
+ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR);
+
+ // do it
+ ZSTD_inBuffer input{ src, complen, 0 };
+ ZSTD_outBuffer output = { dest, destlen, 0 };
+ while ((input.pos < input.size) && (output.pos < output.size))
+ {
+ result = ZSTD_decompressStream(m_stream, &output, &input);
+ if (ZSTD_isError(result))
+ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR);
+ }
+
+ // ensure the expected amount of output was generated
+ if (output.pos != output.size)
+ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR);
+}
+
+
+
+//**************************************************************************
// LZMA ALLOCATOR HELPER
//**************************************************************************
diff --git a/src/lib/util/chdcodec.h b/src/lib/util/chdcodec.h
index 700ae8f8979..c3234ea600d 100644
--- a/src/lib/util/chdcodec.h
+++ b/src/lib/util/chdcodec.h
@@ -155,12 +155,14 @@ constexpr chd_codec_type CHD_CODEC_NONE = 0;
// general codecs
constexpr chd_codec_type CHD_CODEC_ZLIB = CHD_MAKE_TAG('z','l','i','b');
+constexpr chd_codec_type CHD_CODEC_ZSTD = CHD_MAKE_TAG('z','s','t','d');
constexpr chd_codec_type CHD_CODEC_LZMA = CHD_MAKE_TAG('l','z','m','a');
constexpr chd_codec_type CHD_CODEC_HUFFMAN = CHD_MAKE_TAG('h','u','f','f');
constexpr chd_codec_type CHD_CODEC_FLAC = CHD_MAKE_TAG('f','l','a','c');
// general codecs with CD frontend
constexpr chd_codec_type CHD_CODEC_CD_ZLIB = CHD_MAKE_TAG('c','d','z','l');
+constexpr chd_codec_type CHD_CODEC_CD_ZSTD = CHD_MAKE_TAG('c','d','z','s');
constexpr chd_codec_type CHD_CODEC_CD_LZMA = CHD_MAKE_TAG('c','d','l','z');
constexpr chd_codec_type CHD_CODEC_CD_FLAC = CHD_MAKE_TAG('c','d','f','l');
diff --git a/src/lib/util/unzip.cpp b/src/lib/util/unzip.cpp
index 377a1040377..c1df469fed4 100644
--- a/src/lib/util/unzip.cpp
+++ b/src/lib/util/unzip.cpp
@@ -22,6 +22,7 @@
#include "lzma/C/LzmaDec.h"
#include <zlib.h>
+#include <zstd.h>
#include <algorithm>
#include <array>
@@ -289,6 +290,7 @@ private:
std::error_condition decompress_data_type_0(std::uint64_t offset, void *buffer, std::size_t length) noexcept;
std::error_condition decompress_data_type_8(std::uint64_t offset, void *buffer, std::size_t length) noexcept;
std::error_condition decompress_data_type_14(std::uint64_t offset, void *buffer, std::size_t length) noexcept;
+ std::error_condition decompress_data_type_93(std::uint64_t offset, void *buffer, std::size_t length) noexcept;
struct file_header
{
@@ -929,6 +931,9 @@ std::error_condition zip_file_impl::decompress(void *buffer, std::size_t length)
case 14:
return decompress_data_type_14(offset, buffer, length);
+ case 93:
+ return decompress_data_type_93(offset, buffer, length);
+
default:
osd_printf_error(
"unzip: %s in %s uses unsupported compression method %u\n",
@@ -1245,7 +1250,7 @@ std::error_condition zip_file_impl::decompress_data_type_8(std::uint64_t offset,
return archive_file::error::DECOMPRESS_ERROR;
}
};
- std::uint64_t input_remaining = m_header.compressed_length;
+ std::uint64_t input_remaining(m_header.compressed_length);
int zerr;
// reset the stream
@@ -1276,7 +1281,7 @@ std::error_condition zip_file_impl::decompress_data_type_8(std::uint64_t offset,
auto const filerr = m_file->read_at(
offset,
&m_buffer[0],
- std::size_t((std::min<std::uint64_t>)(input_remaining, m_buffer.size())),
+ std::size_t(std::min<std::uint64_t>(input_remaining, m_buffer.size())),
read_length);
if (filerr)
{
@@ -1541,6 +1546,92 @@ std::error_condition zip_file_impl::decompress_data_type_14(std::uint64_t offset
}
}
+
+/*-------------------------------------------------
+ decompress_data_type_93 - decompress
+ type 14 data (Zstandard)
+-------------------------------------------------*/
+
+std::error_condition zip_file_impl::decompress_data_type_93(std::uint64_t offset, void *buffer, std::size_t length) noexcept
+{
+ // create decompression stream
+ ZSTD_DStream *const stream(ZSTD_createDStream());
+ if (!stream)
+ {
+ osd_printf_error(
+ "unzip: error allocating Zstandard stream to decompress %s from %s\n",
+ m_header.file_name, m_filename);
+ return std::errc::not_enough_memory;
+ }
+
+ // loop until we're done
+ std::uint64_t input_remaining(m_header.compressed_length);
+ while (input_remaining && length)
+ {
+ // read in the next chunk of data
+ std::size_t read_length(0);
+ auto const filerr = m_file->read_at(
+ offset,
+ &m_buffer[0],
+ std::size_t(std::min<std::uint64_t>(input_remaining, m_buffer.size())),
+ read_length);
+ if (filerr)
+ {
+ osd_printf_error(
+ "unzip: error reading compressed data for %s in %s (%s:%d %s)\n",
+ m_header.file_name, m_filename, filerr.category().name(), filerr.value(), filerr.message());
+ ZSTD_freeDStream(stream);
+ return filerr;
+ }
+ offset += read_length;
+
+ // if we read nothing, but still have data left, the file is truncated
+ if (!read_length && input_remaining)
+ {
+ osd_printf_error(
+ "unzip: unexpectedly reached end-of-file while reading compressed data for %s in %s\n",
+ m_header.file_name, m_filename);
+ ZSTD_freeDStream(stream);
+ return archive_file::error::FILE_TRUNCATED;
+ }
+
+ // fill out the input data
+ ZSTD_inBuffer input{ &m_buffer[0], read_length, 0 };
+ input_remaining -= read_length;
+
+ // now decompress
+ while ((input.pos < input.size) && length)
+ {
+ ZSTD_outBuffer output{ buffer, length, 0 };
+ auto const result(ZSTD_decompressStream(stream, &output, &input));
+ if (ZSTD_isError(result))
+ {
+ osd_printf_error(
+ "unzip: error decompressing %s from %s (%u: %s)\n",
+ m_header.file_name, m_filename, result, ZSTD_getErrorName(result));
+ ZSTD_freeDStream(stream);
+ return archive_file::error::DECOMPRESS_ERROR;
+ }
+ buffer = reinterpret_cast<std::uint8_t *>(buffer) + output.pos;
+ length -= output.pos;
+ }
+ }
+
+ // free stream
+ ZSTD_freeDStream(stream);
+
+ // if anything looks funny, report an error
+ if (length || input_remaining)
+ {
+ osd_printf_error(
+ "unzip: decompression of %s from %s doesn't appear to have completed correctly\n",
+ m_header.file_name, m_filename);
+ return archive_file::error::DECOMPRESS_ERROR;
+ }
+
+ return std::error_condition();
+}
+
} // anonymous namespace