diff options
Diffstat (limited to 'trunk/src/lib/util/chd.h')
-rw-r--r-- | trunk/src/lib/util/chd.h | 597 |
1 files changed, 597 insertions, 0 deletions
diff --git a/trunk/src/lib/util/chd.h b/trunk/src/lib/util/chd.h new file mode 100644 index 00000000000..67585f2f2ce --- /dev/null +++ b/trunk/src/lib/util/chd.h @@ -0,0 +1,597 @@ +/*************************************************************************** + + chd.h + + MAME Compressed Hunks of Data file format + +**************************************************************************** + + Copyright Aaron Giles + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name 'MAME' nor the names of its contributors may be + used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY AARON GILES ''AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL AARON GILES BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#pragma once + +#ifndef __CHD_H__ +#define __CHD_H__ + +#include "osdcore.h" +#include "coretmpl.h" +#include "astring.h" +#include "bitmap.h" +#include "corefile.h" +#include "hashing.h" +#include "chdcodec.h" + + +/*************************************************************************** + + Compressed Hunks of Data header format. All numbers are stored in + Motorola (big-endian) byte ordering. + + ========================================================================= + + V1 header: + + [ 0] char tag[8]; // 'MComprHD' + [ 8] UINT32 length; // length of header (including tag and length fields) + [ 12] UINT32 version; // drive format version + [ 16] UINT32 flags; // flags (see below) + [ 20] UINT32 compression; // compression type + [ 24] UINT32 hunksize; // 512-byte sectors per hunk + [ 28] UINT32 totalhunks; // total # of hunks represented + [ 32] UINT32 cylinders; // number of cylinders on hard disk + [ 36] UINT32 heads; // number of heads on hard disk + [ 40] UINT32 sectors; // number of sectors on hard disk + [ 44] UINT8 md5[16]; // MD5 checksum of raw data + [ 60] UINT8 parentmd5[16]; // MD5 checksum of parent file + [ 76] (V1 header length) + + Flags: + 0x00000001 - set if this drive has a parent + 0x00000002 - set if this drive allows writes + + Compression types: + CHDCOMPRESSION_NONE = 0 + CHDCOMPRESSION_ZLIB = 1 + + V1 map format: + + [ 0] UINT64 offset : 44; // starting offset within the file + [ 0] UINT64 length : 20; // length of data; if == hunksize, data is uncompressed + + ========================================================================= + + V2 header: + + [ 0] char tag[8]; // 'MComprHD' + [ 8] UINT32 length; // length of header (including tag and length fields) + [ 12] UINT32 version; // drive format version + [ 16] UINT32 flags; // flags (see below) + [ 20] UINT32 compression; // compression type + [ 24] UINT32 hunksize; // seclen-byte sectors per hunk + [ 28] UINT32 totalhunks; // total # of hunks represented + [ 32] UINT32 cylinders; // number of cylinders on hard disk + [ 36] UINT32 heads; // number of heads on hard disk + [ 40] UINT32 sectors; // number of sectors on hard disk + [ 44] UINT8 md5[16]; // MD5 checksum of raw data + [ 60] UINT8 parentmd5[16]; // MD5 checksum of parent file + [ 76] UINT32 seclen; // number of bytes per sector + [ 80] (V2 header length) + + Flags and map format are same as V1 + + ========================================================================= + + V3 header: + + [ 0] char tag[8]; // 'MComprHD' + [ 8] UINT32 length; // length of header (including tag and length fields) + [ 12] UINT32 version; // drive format version + [ 16] UINT32 flags; // flags (see below) + [ 20] UINT32 compression; // compression type + [ 24] UINT32 totalhunks; // total # of hunks represented + [ 28] UINT64 logicalbytes; // logical size of the data (in bytes) + [ 36] UINT64 metaoffset; // offset to the first blob of metadata + [ 44] UINT8 md5[16]; // MD5 checksum of raw data + [ 60] UINT8 parentmd5[16]; // MD5 checksum of parent file + [ 76] UINT32 hunkbytes; // number of bytes per hunk + [ 80] UINT8 sha1[20]; // SHA1 checksum of raw data + [100] UINT8 parentsha1[20];// SHA1 checksum of parent file + [120] (V3 header length) + + Flags are the same as V1 + + Compression types: + CHDCOMPRESSION_NONE = 0 + CHDCOMPRESSION_ZLIB = 1 + CHDCOMPRESSION_ZLIB_PLUS = 2 + + V3 map format: + + [ 0] UINT64 offset; // starting offset within the file + [ 8] UINT32 crc32; // 32-bit CRC of the uncompressed data + [ 12] UINT16 length_lo; // lower 16 bits of length + [ 14] UINT8 length_hi; // upper 8 bits of length + [ 15] UINT8 flags; // flags, indicating compression info + + ========================================================================= + + V4 header: + + [ 0] char tag[8]; // 'MComprHD' + [ 8] UINT32 length; // length of header (including tag and length fields) + [ 12] UINT32 version; // drive format version + [ 16] UINT32 flags; // flags (see below) + [ 20] UINT32 compression; // compression type + [ 24] UINT32 totalhunks; // total # of hunks represented + [ 28] UINT64 logicalbytes; // logical size of the data (in bytes) + [ 36] UINT64 metaoffset; // offset to the first blob of metadata + [ 44] UINT32 hunkbytes; // number of bytes per hunk + [ 48] UINT8 sha1[20]; // combined raw+meta SHA1 + [ 68] UINT8 parentsha1[20];// combined raw+meta SHA1 of parent + [ 88] UINT8 rawsha1[20]; // raw data SHA1 + [108] (V4 header length) + + Flags are the same as V1 + + Compression types: + CHDCOMPRESSION_NONE = 0 + CHDCOMPRESSION_ZLIB = 1 + CHDCOMPRESSION_ZLIB_PLUS = 2 + CHDCOMPRESSION_AV = 3 + + Map format is the same as V3 + + ========================================================================= + + V5 header: + + [ 0] char tag[8]; // 'MComprHD' + [ 8] UINT32 length; // length of header (including tag and length fields) + [ 12] UINT32 version; // drive format version + [ 16] UINT32 compressors[4];// which custom compressors are used? + [ 32] UINT64 logicalbytes; // logical size of the data (in bytes) + [ 40] UINT64 mapoffset; // offset to the map + [ 48] UINT64 metaoffset; // offset to the first blob of metadata + [ 56] UINT32 hunkbytes; // number of bytes per hunk (512k maximum) + [ 60] UINT32 unitbytes; // number of bytes per unit within each hunk + [ 64] UINT8 rawsha1[20]; // raw data SHA1 + [ 84] UINT8 sha1[20]; // combined raw+meta SHA1 + [104] UINT8 parentsha1[20];// combined raw+meta SHA1 of parent + [124] (V5 header length) + + If parentsha1 != 0, we have a parent (no need for flags) + If compressors[0] == 0, we are uncompressed (including maps) + + V5 uncompressed map format: + + [ 0] UINT32 offset; // starting offset / hunk size + + V5 compressed map format header: + + [ 0] UINT32 length; // length of compressed map + [ 4] UINT48 datastart; // offset of first block + [ 10] UINT16 crc; // crc-16 of the map + [ 12] UINT8 lengthbits; // bits used to encode complength + [ 13] UINT8 hunkbits; // bits used to encode self-refs + [ 14] UINT8 parentunitbits; // bits used to encode parent unit refs + [ 15] UINT8 reserved; // future use + [ 16] (compressed header length) + + Each compressed map entry, once expanded, looks like: + + [ 0] UINT8 compression; // compression type + [ 1] UINT24 complength; // compressed length + [ 4] UINT48 offset; // offset + [ 10] UINT16 crc; // crc-16 of the data + +***************************************************************************/ + + +//************************************************************************** +// CONSTANTS +//************************************************************************** + +// pseudo-codecs returned by hunk_info +const chd_codec_type CHD_CODEC_SELF = 1; // copy of another hunk +const chd_codec_type CHD_CODEC_PARENT = 2; // copy of a parent's hunk +const chd_codec_type CHD_CODEC_MINI = 3; // legacy "mini" 8-byte repeat + +// core types +typedef UINT32 chd_metadata_tag; + +// metadata parameters +const chd_metadata_tag CHDMETATAG_WILDCARD = 0; +const UINT32 CHDMETAINDEX_APPEND = ~0; + +// metadata flags +const UINT8 CHD_MDFLAGS_CHECKSUM = 0x01; // indicates data is checksummed + +// standard hard disk metadata +const chd_metadata_tag HARD_DISK_METADATA_TAG = CHD_MAKE_TAG('G','D','D','D'); +extern const char *HARD_DISK_METADATA_FORMAT; + +// hard disk identify information +const chd_metadata_tag HARD_DISK_IDENT_METADATA_TAG = CHD_MAKE_TAG('I','D','N','T'); + +// hard disk key information +const chd_metadata_tag HARD_DISK_KEY_METADATA_TAG = CHD_MAKE_TAG('K','E','Y',' '); + +// pcmcia CIS information +const chd_metadata_tag PCMCIA_CIS_METADATA_TAG = CHD_MAKE_TAG('C','I','S',' '); + +// standard CD-ROM metadata +const chd_metadata_tag CDROM_OLD_METADATA_TAG = CHD_MAKE_TAG('C','H','C','D'); +const chd_metadata_tag CDROM_TRACK_METADATA_TAG = CHD_MAKE_TAG('C','H','T','R'); +extern const char *CDROM_TRACK_METADATA_FORMAT; +const chd_metadata_tag CDROM_TRACK_METADATA2_TAG = CHD_MAKE_TAG('C','H','T','2'); +extern const char *CDROM_TRACK_METADATA2_FORMAT; +const chd_metadata_tag GDROM_TRACK_METADATA_TAG = CHD_MAKE_TAG('C','H','G','T'); +extern const char *GDROM_TRACK_METADATA_FORMAT; + +// standard A/V metadata +const chd_metadata_tag AV_METADATA_TAG = CHD_MAKE_TAG('A','V','A','V'); +extern const char *AV_METADATA_FORMAT; + +// A/V laserdisc frame metadata +const chd_metadata_tag AV_LD_METADATA_TAG = CHD_MAKE_TAG('A','V','L','D'); + +// error types +enum chd_error +{ + CHDERR_NONE, + CHDERR_NO_INTERFACE, + CHDERR_OUT_OF_MEMORY, + CHDERR_NOT_OPEN, + CHDERR_ALREADY_OPEN, + CHDERR_INVALID_FILE, + CHDERR_INVALID_PARAMETER, + CHDERR_INVALID_DATA, + CHDERR_FILE_NOT_FOUND, + CHDERR_REQUIRES_PARENT, + CHDERR_FILE_NOT_WRITEABLE, + CHDERR_READ_ERROR, + CHDERR_WRITE_ERROR, + CHDERR_CODEC_ERROR, + CHDERR_INVALID_PARENT, + CHDERR_HUNK_OUT_OF_RANGE, + CHDERR_DECOMPRESSION_ERROR, + CHDERR_COMPRESSION_ERROR, + CHDERR_CANT_CREATE_FILE, + CHDERR_CANT_VERIFY, + CHDERR_NOT_SUPPORTED, + CHDERR_METADATA_NOT_FOUND, + CHDERR_INVALID_METADATA_SIZE, + CHDERR_UNSUPPORTED_VERSION, + CHDERR_VERIFY_INCOMPLETE, + CHDERR_INVALID_METADATA, + CHDERR_INVALID_STATE, + CHDERR_OPERATION_PENDING, + CHDERR_UNSUPPORTED_FORMAT, + CHDERR_UNKNOWN_COMPRESSION, + CHDERR_WALKING_PARENT, + CHDERR_COMPRESSING +}; + + + +//************************************************************************** +// TYPE DEFINITIONS +//************************************************************************** + +class chd_codec; + + +// ======================> chd_file + +// core file class +class chd_file +{ + friend class chd_file_compressor; + friend class chd_verifier; + + // constants + static const UINT32 HEADER_VERSION = 5; + static const UINT32 V3_HEADER_SIZE = 120; + static const UINT32 V4_HEADER_SIZE = 108; + static const UINT32 V5_HEADER_SIZE = 124; + static const UINT32 MAX_HEADER_SIZE = V5_HEADER_SIZE; + +public: + // construction/destruction + chd_file(); + virtual ~chd_file(); + + // operators + operator core_file *() { return m_file; } + + // getters + bool opened() const { return (m_file != NULL); } + UINT32 version() const { return m_version; } + UINT64 logical_bytes() const { return m_logicalbytes; } + UINT32 hunk_bytes() const { return m_hunkbytes; } + UINT32 hunk_count() const { return m_hunkcount; } + UINT32 unit_bytes() const { return m_unitbytes; } + UINT64 unit_count() const { return m_unitcount; } + bool compressed() const { return (m_compression[0] != CHD_CODEC_NONE); } + chd_codec_type compression(int index) const { return m_compression[index]; } + chd_file *parent() const { return m_parent; } + sha1_t sha1(); + sha1_t raw_sha1(); + sha1_t parent_sha1(); + chd_error hunk_info(UINT32 hunknum, chd_codec_type &compressor, UINT32 &compbytes); + + // setters + void set_raw_sha1(sha1_t rawdata); + void set_parent_sha1(sha1_t parent); + + // file create + chd_error create(const char *filename, UINT64 logicalbytes, UINT32 hunkbytes, UINT32 unitbytes, chd_codec_type compression[4]); + chd_error create(core_file &file, UINT64 logicalbytes, UINT32 hunkbytes, UINT32 unitbytes, chd_codec_type compression[4]); + chd_error create(const char *filename, UINT64 logicalbytes, UINT32 hunkbytes, chd_codec_type compression[4], chd_file &parent); + chd_error create(core_file &file, UINT64 logicalbytes, UINT32 hunkbytes, chd_codec_type compression[4], chd_file &parent); + + // file open + chd_error open(const char *filename, bool writeable = false, chd_file *parent = NULL); + chd_error open(core_file &file, bool writeable = false, chd_file *parent = NULL); + + // file close + void close(); + + // read/write + chd_error read_hunk(UINT32 hunknum, void *buffer); + chd_error write_hunk(UINT32 hunknum, const void *buffer); + chd_error read_units(UINT64 unitnum, void *buffer, UINT32 count = 1); + chd_error write_units(UINT64 unitnum, const void *buffer, UINT32 count = 1); + chd_error read_bytes(UINT64 offset, void *buffer, UINT32 bytes); + chd_error write_bytes(UINT64 offset, const void *buffer, UINT32 bytes); + + // metadata management + chd_error read_metadata(chd_metadata_tag searchtag, UINT32 searchindex, astring &output); + chd_error read_metadata(chd_metadata_tag searchtag, UINT32 searchindex, dynamic_buffer &output); + chd_error read_metadata(chd_metadata_tag searchtag, UINT32 searchindex, void *output, UINT32 outputlen, UINT32 &resultlen); + chd_error read_metadata(chd_metadata_tag searchtag, UINT32 searchindex, dynamic_buffer &output, chd_metadata_tag &resulttag, UINT8 &resultflags); + chd_error write_metadata(chd_metadata_tag metatag, UINT32 metaindex, const void *inputbuf, UINT32 inputlen, UINT8 flags = CHD_MDFLAGS_CHECKSUM); + chd_error write_metadata(chd_metadata_tag metatag, UINT32 metaindex, const astring &input, UINT8 flags = CHD_MDFLAGS_CHECKSUM) { return write_metadata(metatag, metaindex, input.cstr(), input.len() + 1, flags); } + chd_error write_metadata(chd_metadata_tag metatag, UINT32 metaindex, const dynamic_buffer &input, UINT8 flags = CHD_MDFLAGS_CHECKSUM) { return write_metadata(metatag, metaindex, input, input.count(), flags); } + chd_error delete_metadata(chd_metadata_tag metatag, UINT32 metaindex); + chd_error clone_all_metadata(chd_file &source); + + // hashing helper + sha1_t compute_overall_sha1(sha1_t rawsha1); + + // codec interfaces + chd_error codec_configure(chd_codec_type codec, int param, void *config); + + // static helpers + static const char *error_string(chd_error err); + +private: + struct metadata_entry; + struct metadata_hash; + + // inline helpers + UINT64 be_read(const UINT8 *base, int numbytes); + void be_write(UINT8 *base, UINT64 value, int numbytes); + sha1_t be_read_sha1(const UINT8 *base); + void be_write_sha1(UINT8 *base, sha1_t value); + void file_read(UINT64 offset, void *dest, UINT32 length); + void file_write(UINT64 offset, const void *source, UINT32 length); + UINT64 file_append(const void *source, UINT32 length, UINT32 alignment = 0); + UINT8 bits_for_value(UINT64 value); + + // internal helpers + UINT32 guess_unitbytes(); + void parse_v3_header(UINT8 *rawheader, sha1_t &parentsha1); + void parse_v4_header(UINT8 *rawheader, sha1_t &parentsha1); + void parse_v5_header(UINT8 *rawheader, sha1_t &parentsha1); + chd_error compress_v5_map(); + void decompress_v5_map(); + chd_error create_common(); + chd_error open_common(bool writeable); + void create_open_common(); + void verify_proper_compression_append(UINT32 hunknum); + void hunk_write_compressed(UINT32 hunknum, INT8 compression, const UINT8 *compressed, UINT32 complength, crc16_t crc16); + void hunk_copy_from_self(UINT32 hunknum, UINT32 otherhunk); + void hunk_copy_from_parent(UINT32 hunknum, UINT64 parentunit); + bool metadata_find(chd_metadata_tag metatag, INT32 metaindex, metadata_entry &metaentry, bool resume = false); + void metadata_set_previous_next(UINT64 prevoffset, UINT64 nextoffset); + void metadata_update_hash(); + static int CLIB_DECL metadata_hash_compare(const void *elem1, const void *elem2); + + // file characteristics + core_file * m_file; // handle to the open core file + bool m_owns_file; // flag indicating if this file should be closed on chd_close() + bool m_allow_reads; // permit reads from this CHD? + bool m_allow_writes; // permit writes to this CHD? + + // core parameters from the header + UINT32 m_version; // version of the header + UINT64 m_logicalbytes; // logical size of the raw CHD data in bytes + UINT64 m_mapoffset; // offset of map + UINT64 m_metaoffset; // offset to first metadata bit + UINT32 m_hunkbytes; // size of each raw hunk in bytes + UINT32 m_hunkcount; // number of hunks represented + UINT32 m_unitbytes; // size of each unit in bytes + UINT64 m_unitcount; // number of units represented + chd_codec_type m_compression[4]; // array of compression types used + chd_file * m_parent; // pointer to parent file, or NULL if none + bool m_parent_missing; // are we missing our parent? + + // key offsets within the header + UINT64 m_mapoffset_offset; // offset of map offset field + UINT64 m_metaoffset_offset;// offset of metaoffset field + UINT64 m_sha1_offset; // offset of SHA1 field + UINT64 m_rawsha1_offset; // offset of raw SHA1 field + UINT64 m_parentsha1_offset;// offset of paren SHA1 field + + // map information + UINT32 m_mapentrybytes; // length of each entry in a map + dynamic_buffer m_rawmap; // raw map data + + // compression management + chd_decompressor * m_decompressor[4]; // array of decompression codecs + dynamic_buffer m_compressed; // temporary buffer for compressed data + + // caching + dynamic_buffer m_cache; // single-hunk cache for partial reads/writes + UINT32 m_cachehunk; // which hunk is in the cache? +}; + + +// ======================> chd_file_compressor + +// class for creating a new compressed CHD +class chd_file_compressor : public chd_file +{ +public: + // construction/destruction + chd_file_compressor(); + virtual ~chd_file_compressor(); + + // compression management + void compress_begin(); + chd_error compress_continue(double &progress, double &ratio); + +protected: + // required override: read more data + virtual UINT32 read_data(void *dest, UINT64 offset, UINT32 length) = 0; + +private: + // hash map for looking up values + class hashmap + { + public: + // construction/destruction + hashmap(); + ~hashmap(); + + // operations + void reset(); + UINT64 find(crc16_t crc16, sha1_t sha1); + void add(UINT64 itemnum, crc16_t crc16, sha1_t sha1); + + // constants + static const UINT64 NOT_FOUND = ~UINT64(0); + private: + // internal entry + struct entry_t + { + entry_t * m_next; // next entry in list + UINT64 m_itemnum; // item number + sha1_t m_sha1; // SHA-1 of the block + }; + + // block of entries + struct entry_block + { + entry_block(entry_block *prev) + : m_next(prev), m_nextalloc(0) { } + + entry_block * m_next; // next block in list + UINT32 m_nextalloc; // next to be allocated + entry_t m_array[16384]; // array of entries + }; + + // internal state + entry_t * m_map[65536]; // map, hashed by CRC-16 + entry_block * m_block_list; // list of allocated blocks + }; + + // status of a given work item + enum work_status + { + WS_READY = 0, + WS_READING, + WS_QUEUED, + WS_COMPLETE + }; + + // a CRC-16/SHA-1 pair + struct hash_pair + { + crc16_t m_crc16; // calculated CRC-16 + sha1_t m_sha1; // calculated SHA-1 + }; + + // a single work item + struct work_item + { + osd_work_item * m_osd; // OSD work item running on this block + chd_file_compressor *m_compressor; // pointer back to the compressor + volatile work_status m_status; // current status of this item + UINT32 m_hunknum; // number of the hunk we're working on + UINT8 * m_data; // pointer to the data we are working on + UINT8 * m_compressed; // pointer to the compressed data + UINT32 m_complen; // compressed data length + INT8 m_compression; // type of compression used + chd_compressor_group *m_codecs; // codec instance + dynamic_array<hash_pair> m_hash; // array of hashes + }; + + // internal helpers + static void *async_walk_parent_static(void *param, int threadid); + void async_walk_parent(work_item &item); + static void *async_compress_hunk_static(void *param, int threadid); + void async_compress_hunk(work_item &item, int threadid); + static void *async_read_static(void *param, int threadid); + void async_read(); + + // current compression status + bool m_walking_parent; // are we building the parent map? + UINT64 m_total_in; // total bytes in + UINT64 m_total_out; // total bytes out + sha1_creator m_compsha1; // running SHA-1 on raw data + + // hash lookup maps + hashmap m_parent_map; // hash map for parent + hashmap m_current_map; // hash map for current + + // read I/O thread + osd_work_queue * m_read_queue; // work queue for reading + UINT64 m_read_queue_offset;// next offset to enqueue + UINT64 m_read_done_offset; // next offset that will complete + bool m_read_error; // error during reading? + + // work item thread + static const int WORK_BUFFER_HUNKS = 256; + osd_work_queue * m_work_queue; // queue for doing work on other threads + dynamic_buffer m_work_buffer; // buffer containing hunk data to work on + dynamic_buffer m_compressed_buffer;// buffer containing compressed data + work_item m_work_item[WORK_BUFFER_HUNKS]; // status of each hunk + chd_compressor_group * m_codecs[WORK_MAX_THREADS]; // codecs to use + + // output state + UINT32 m_write_hunk; // next hunk to write +}; + + +#endif // __CHD_H__ |