summaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
author Aaron Giles <aaron@aarongiles.com>2021-04-16 02:38:34 -0700
committer Aaron Giles <aaron@aarongiles.com>2021-04-16 02:38:34 -0700
commit13aee47f5fdba9a337d9f3e1490e265b706cbd9f (patch)
tree3deb436c6b5ab31b682eff7e2acd58f94608aa99
parent2024b61079509b9f1ef692ff3f9e63007a831f6f (diff)
Implemented save_file. JSON and secondary files are now written to a ZIP-formatted file.
-rw-r--r--src/emu/save.cpp472
-rw-r--r--src/emu/save.h39
2 files changed, 463 insertions, 48 deletions
diff --git a/src/emu/save.cpp b/src/emu/save.cpp
index 418d914f2ff..9a91c784b16 100644
--- a/src/emu/save.cpp
+++ b/src/emu/save.cpp
@@ -27,6 +27,7 @@
#include "coreutil.h"
#include <iomanip>
+#include <zlib.h>
//**************************************************************************
@@ -257,7 +258,7 @@ save_registered_item *save_registered_item::find(char const *name)
bool save_registered_item::sort_and_prune()
{
// only applies to arrays, structs, and containers; don't prune anything else
- if (m_type >= TYPE_ARRAY && m_type != TYPE_STRUCT && m_type != TYPE_CONTAINER)
+ if (!is_array() && !is_struct_or_container())
return false;
// first prune any empty items
@@ -358,7 +359,7 @@ uint64_t save_registered_item::save_binary(uint8_t *ptr, uint64_t length, uintpt
// arrays are multiples of a single item
default:
- if (m_type < TYPE_ARRAY)
+ if (is_array())
{
auto &item = m_items.front();
for (uint32_t rep = 0; rep < m_type; rep++)
@@ -410,7 +411,7 @@ uint64_t save_registered_item::restore_binary(uint8_t const *ptr, uint64_t lengt
// arrays are multiples of a single item
default:
- if (m_type < TYPE_ARRAY)
+ if (is_array())
{
auto &item = m_items.front();
for (uint32_t rep = 0; rep < m_type; rep++)
@@ -500,35 +501,25 @@ void save_registered_item::save_json(save_zip_state &zipstate, char const *namep
// arrays are multiples of a single item
default:
- if (m_type < TYPE_ARRAY)
+ if (is_array())
{
auto &item = m_items.front();
// look for large arrays of ints/floats
save_registered_item *inner = &item;
u32 total = count();
- while (inner->type() < TYPE_ARRAY)
+ while (inner->is_array())
{
total *= inner->count();
inner = &inner->m_items.front();
}
- if ((inner->type() == TYPE_INT || inner->type() == TYPE_UINT || inner->type() == TYPE_FLOAT) && total * inner->m_native_size >= save_zip_state::JSON_EXTERNAL_BINARY_THRESHOLD)
+ if (inner->is_int_or_float() && total * inner->m_native_size >= save_zip_state::JSON_EXTERNAL_BINARY_THRESHOLD)
{
- std::string filename = localname;
- for (int index = 0; index < filename.length(); )
- if (strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.", filename[index]) == nullptr)
- {
- if (index != 0 && filename[index - 1] != '.')
- filename[index++] = '.';
- else
- filename.erase(index, 1);
- }
- else
- index++;
+ char const *filename = zipstate.add_data_file(localname.c_str(), *this, objbase);
zipstate.json_append('[').json_append('{');
zipstate.json_append_name("external_file");
- zipstate.json_append('"').json_append(filename.c_str()).json_append('"').json_append(',');
+ zipstate.json_append('"').json_append(filename).json_append('"').json_append(',');
zipstate.json_append_name("unit");
zipstate.json_append_signed(inner->m_native_size).json_append(',');
zipstate.json_append_name("count");
@@ -536,8 +527,6 @@ void save_registered_item::save_json(save_zip_state &zipstate, char const *namep
zipstate.json_append_name("little_endian");
zipstate.json_append((ENDIANNESS_NATIVE == ENDIANNESS_LITTLE) ? "true" : "false");
zipstate.json_append('}').json_append(']');
-
- zipstate.add_data_file(filename.c_str(), item, reinterpret_cast<void *>(objbase));
}
else
{
@@ -558,11 +547,9 @@ void save_registered_item::save_json(save_zip_state &zipstate, char const *namep
{
// normal form outputs a certain number of items per row
zipstate.json_append('[').json_append_eol();
- uint32_t items_per_row = 0;
- if (item.m_type == TYPE_INT || item.m_type == TYPE_UINT || item.m_type == TYPE_FLOAT)
- items_per_row = 32 / item_size;
- if (items_per_row == 0)
- items_per_row = 1;
+ uint32_t items_per_row = 1;
+ if (item.is_int_or_float())
+ items_per_row = (item_size <= 2) ? 32 : 16;
// iterate over the items
for (uint32_t rep = 0; rep < m_type; rep++)
@@ -833,10 +820,8 @@ save_error save_manager::save_file(emu_file &file)
save_zip_state state;
m_root_item.save_json(state);
- // write the output
- __debugbreak();
-
- return STATERR_NONE;
+ // then commit the state to the file
+ return state.commit(file) ? STATERR_NONE : STATERR_WRITE_ERROR;
}
@@ -1221,13 +1206,100 @@ void rewinder::report_error(save_error error, rewind_operation operation)
}
-void save_manager::test_dump()
+
+//**************************************************************************
+// ZLIB STREAMER
+//**************************************************************************
+
+class zlib_streamer
{
- save_zip_state state;
- m_root_item.save_json(state);
- printf("%s\n", state.json_string());
-}
+public:
+ // construction
+ zlib_streamer(emu_file &output) :
+ m_output(output)
+ {
+ m_stream.zalloc = Z_NULL;
+ m_stream.zfree = Z_NULL;
+ m_stream.opaque = Z_NULL;
+ m_stream.avail_in = m_stream.avail_out = 0;
+ }
+ // initialize compression
+ bool begin()
+ {
+ // reset the output buffer
+ m_stream.next_out = &m_buffer[0];
+ m_stream.avail_out = sizeof(m_buffer);
+
+ // initialize the zlib engine; the negative window size means
+ // no headers, which is what a .ZIP file wants
+ return (deflateInit2(&m_stream, Z_BEST_COMPRESSION, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) == Z_OK);
+ }
+
+ // add more compressed data
+ bool write(void const *data, u32 count)
+ {
+ // point the input buffer to the data
+ m_stream.next_in = const_cast<Bytef *>(reinterpret_cast<Bytef const *>(data));
+ m_stream.avail_in = count;
+
+ // loop until all consumed
+ while (m_stream.avail_in != 0)
+ {
+ // deflate as much as possible
+ if (deflate(&m_stream, Z_NO_FLUSH) != Z_OK)
+ return false;
+
+ // if we ran out of output space, flush to the file and reset
+ if (m_stream.avail_out == 0)
+ {
+ m_output.write(&m_buffer[0], sizeof(m_buffer));
+ m_stream.next_out = &m_buffer[0];
+ m_stream.avail_out = sizeof(m_buffer);
+ }
+ }
+ return true;
+ }
+
+ // finish cmopression
+ bool end()
+ {
+ // loop until all data processed
+ int zerr = Z_OK;
+ while (zerr != Z_STREAM_END)
+ {
+ // deflate and attempt to finish
+ zerr = deflate(&m_stream, Z_FINISH);
+ if (zerr != Z_OK && zerr != Z_STREAM_END)
+ return false;
+
+ // if there's any output data, flush it to the file and reset
+ if (m_stream.avail_out != sizeof(m_buffer))
+ {
+ m_output.write(&m_buffer[0], sizeof(m_buffer) - m_stream.avail_out);
+ m_stream.next_out = &m_buffer[0];
+ m_stream.avail_out = sizeof(m_buffer);
+ }
+ }
+ return true;
+ }
+
+private:
+ // internal state
+ emu_file &m_output;
+ z_stream m_stream;
+ u8 m_buffer[4096];
+};
+
+
+
+//**************************************************************************
+// SAVE ZIP STATE
+//**************************************************************************
+
+//-------------------------------------------------
+// save_zip_state - constuctor
+//-------------------------------------------------
save_zip_state::save_zip_state() :
m_json_reserved(0),
@@ -1236,7 +1308,335 @@ save_zip_state::save_zip_state() :
json_check_reserve();
}
-void save_zip_state::commit(FILE &output)
+
+//-------------------------------------------------
+// add_data_file - add a data file to the ZIP
+// file, creating a clean, unique filename for it
+//-------------------------------------------------
+
+char const *save_zip_state::add_data_file(char const *proposed_name, save_registered_item &item, uintptr_t base)
+{
+ // first sanitize the filename
+ std::string base_filename = proposed_name;
+ for (int index = 0; index < base_filename.length(); )
+ {
+ if (strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.", base_filename[index]) == nullptr)
+ {
+ if (index != 0 && base_filename[index - 1] != '.')
+ base_filename[index++] = '.';
+ else
+ base_filename.erase(index, 1);
+ }
+ else
+ index++;
+ }
+
+ // now ensure it is unique
+ std::string filename;
+ bool retry = true;
+ for (int index = 1; retry; index++)
+ {
+ if (index == 1)
+ filename = string_format("%s.bin", base_filename.c_str());
+ else
+ filename = string_format("%s.%d.bin", base_filename.c_str(), index);
+
+ // see if anyone else has this name; if so, retry it
+ retry = false;
+ for (auto &file : m_file_list)
+ if (filename == file.m_name)
+ {
+ retry = true;
+ break;
+ }
+ }
+
+ // add to the list
+ m_file_list.emplace_back(filename.c_str(), item, base);
+ return m_file_list.back().m_name.c_str();
+}
+
+
+//-------------------------------------------------
+// commit - assemble all the files into their
+// final forms and write the ZIP data to the
+// output file
+//-------------------------------------------------
+
+bool save_zip_state::commit(emu_file &output)
+{
+ zlib_streamer zlib(output);
+ std::vector<u8> local_header;
+ std::vector<u8> local_footer;
+
+ // determine the MS-DOS formatted time
+ time_t rawtime;
+ ::time(&rawtime);
+ struct tm &timeinfo = *localtime(&rawtime);
+ m_archive_date = timeinfo.tm_mday | ((timeinfo.tm_mon + 1) << 5) | ((timeinfo.tm_year - 1980) << 9);
+ m_archive_time = (timeinfo.tm_sec / 2) | (timeinfo.tm_min << 5) | (timeinfo.tm_hour << 11);
+
+ // write the local header (and create the central directory entry) for the JSON itself
+ std::vector<u8> json_central_directory;
+ u64 local_header_offset = output.tell();
+ create_zip_file_header(local_header, json_central_directory, "save.json", local_header_offset);
+ output.write(&local_header[0], local_header.size());
+
+ // stream the JSON and compress it
+ u64 start = output.tell();
+ if (!zlib.begin() || !zlib.write(&m_json[0], m_json_offset) || !zlib.end())
+ return false;
+
+ // write the local footer and update the central directory entry
+ create_zip_file_footer(local_footer, json_central_directory, m_json_offset, output.tell() - start, util::crc32_creator::simple(&m_json[0], m_json_offset));
+ output.seek(local_header_offset + 0xe, SEEK_SET);
+ output.write(&local_footer[0], local_footer.size());
+ output.seek(0, SEEK_END);
+
+ // then write out the other files
+ for (auto &file : m_file_list)
+ {
+ // reset the accumulators
+ m_file_crc_accum.reset();
+ m_file_size_accum = 0;
+
+ // write the local header (and create the central directory entry) for the file
+ u64 local_header_offset = output.tell();
+ create_zip_file_header(local_header, file.m_central_directory, file.m_name.c_str(), local_header_offset);
+ output.write(&local_header[0], local_header.size());
+
+ // write the file header and compress it
+ u64 start = output.tell();
+ if (!zlib.begin() || !write_data_recursive(zlib, file.m_item, file.m_base) || !zlib.end())
+ return false;
+
+ // write the local footer and update the central directory entry
+ create_zip_file_footer(local_footer, file.m_central_directory, m_file_size_accum, output.tell() - start, m_file_crc_accum.finish());
+ output.seek(local_header_offset + 0xe, SEEK_SET);
+ output.write(&local_footer[0], local_footer.size());
+ output.seek(0, SEEK_END);
+ }
+
+ // remember the base of the central directory, then write it
+ u64 central_dir_offset = output.tell();
+ output.write(&json_central_directory[0], json_central_directory.size());
+ for (auto &file : m_file_list)
+ output.write(&file.m_central_directory[0], file.m_central_directory.size());
+
+ // now create the
+ std::vector<u8> eocd;
+ create_end_of_central_directory(eocd, m_file_list.size() + 1, central_dir_offset, output.tell() - central_dir_offset);
+ output.write(&eocd[0], eocd.size());
+ return true;
+}
+
+
+//-------------------------------------------------
+// create_zip_file_header - create both the local
+// and central file headers; the CRC and size
+// information is stored as 0 at this stage
+//-------------------------------------------------
+
+void save_zip_state::create_zip_file_header(std::vector<u8> &local, std::vector<u8> &central, char const *filename, u64 local_offset)
+{
+ // reset the headers
+ local.clear();
+ central.clear();
+
+ // write the standard headers
+ local.push_back(0x50); central.push_back(0x50);
+ local.push_back(0x4b); central.push_back(0x4b);
+ local.push_back(0x03); central.push_back(0x01);
+ local.push_back(0x04); central.push_back(0x02);
+
+ // version created by = 3.0 / 0 (MS-DOS) (central directory only)
+ central.push_back(0x1e);
+ central.push_back(0x00);
+
+ // version to extract = 2.0
+ local.push_back(0x14); central.push_back(0x14);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // general purpose bit flag = 0x02 (2=max compression)
+ local.push_back(0x02); central.push_back(0x02);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // compression method = 8 (deflate)
+ local.push_back(0x08); central.push_back(0x08);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // last mod file time
+ local.push_back(BIT(m_archive_time, 0, 8)); central.push_back(BIT(m_archive_time, 0, 8));
+ local.push_back(BIT(m_archive_time, 8, 8)); central.push_back(BIT(m_archive_time, 8, 8));
+
+ // last mod file date
+ local.push_back(BIT(m_archive_date, 0, 8)); central.push_back(BIT(m_archive_date, 0, 8));
+ local.push_back(BIT(m_archive_date, 8, 8)); central.push_back(BIT(m_archive_date, 8, 8));
+
+ // crc-32 -- to be written later
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // compressed size -- to be written later
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // uncompressed size -- to be written later
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // file name length
+ u16 len = strlen(filename);
+ local.push_back(BIT(len, 0, 8)); central.push_back(BIT(len, 0, 8));
+ local.push_back(BIT(len, 8, 8)); central.push_back(BIT(len, 8, 8));
+
+ // extra field length
+ local.push_back(0x00); central.push_back(0x00);
+ local.push_back(0x00); central.push_back(0x00);
+
+ // file comment length (central directory only)
+ central.push_back(0x00);
+ central.push_back(0x00);
+
+ // disk number start (central directory only)
+ central.push_back(0x00);
+ central.push_back(0x00);
+
+ // internal file attributes (central directory only)
+ central.push_back(0x00);
+ central.push_back(0x00);
+
+ // external file attributes (central directory only)
+ central.push_back(0x00);
+ central.push_back(0x00);
+ central.push_back(0x00);
+ central.push_back(0x00);
+
+ // relative offset of local header (central directory only)
+ central.push_back(BIT(local_offset, 0, 8));
+ central.push_back(BIT(local_offset, 8, 8));
+ central.push_back(BIT(local_offset, 16, 8));
+ central.push_back(BIT(local_offset, 24, 8));
+
+ // filename
+ for ( ; *filename != 0; filename++)
+ {
+ local.push_back(*filename);
+ central.push_back(*filename);
+ }
+}
+
+
+//-------------------------------------------------
+// create_zip_file_footer - create the CRC and
+// size information, and update the central
+// directory entry with the data
+//-------------------------------------------------
+
+void save_zip_state::create_zip_file_footer(std::vector<u8> &local, std::vector<u8> &central, u32 filesize, u32 compressed, u32 crc)
+{
+ // reset the local footer data
+ local.clear();
+
+ // crc-32 -- to be written later
+ local.push_back(central[16] = BIT(crc, 0, 8));
+ local.push_back(central[17] = BIT(crc, 8, 8));
+ local.push_back(central[18] = BIT(crc, 16, 8));
+ local.push_back(central[19] = BIT(crc, 24, 8));
+
+ // compressed size -- to be written later
+ local.push_back(central[20] = BIT(compressed, 0, 8));
+ local.push_back(central[21] = BIT(compressed, 8, 8));
+ local.push_back(central[22] = BIT(compressed, 16, 8));
+ local.push_back(central[23] = BIT(compressed, 24, 8));
+
+ // uncompressed size -- to be written later
+ local.push_back(central[24] = BIT(filesize, 0, 8));
+ local.push_back(central[25] = BIT(filesize, 8, 8));
+ local.push_back(central[26] = BIT(filesize, 16, 8));
+ local.push_back(central[27] = BIT(filesize, 24, 8));
+}
+
+
+//-------------------------------------------------
+// write_data_recursive - write potentially
+// multi-dimensional arrays to the compressed
+// output, computing size and CRC
+//-------------------------------------------------
+
+bool save_zip_state::write_data_recursive(zlib_streamer &zlib, save_registered_item &item, uintptr_t base)
+{
+ save_registered_item &inner = item.subitems().front();
+ if (inner.is_array())
+ {
+ for (int index = 0; index < item.count(); index++)
+ {
+ if (!write_data_recursive(zlib, inner, base))
+ return false;
+ base += item.native_size();
+ }
+ }
+ else
+ {
+ u32 size = item.count() * item.native_size();
+ if (!zlib.write(reinterpret_cast<void *>(base), size))
+ return false;
+ m_file_crc_accum.append(reinterpret_cast<void *>(base), size);
+ m_file_size_accum += size;
+ }
+ return true;
+}
+
+
+//-------------------------------------------------
+// create_end_of_central_directory - create a
+// buffer containing the end of central directory
+// record
+//-------------------------------------------------
+
+void save_zip_state::create_end_of_central_directory(std::vector<u8> &header, u32 central_dir_entries, u64 central_dir_offset, u32 central_dir_size)
{
-// add_data_file("save.json", &m_json.str()[0], m_json.str().length());
+ // end of central directory header
+ header.push_back(0x50);
+ header.push_back(0x4b);
+ header.push_back(0x05);
+ header.push_back(0x06);
+
+ // number of this disk
+ header.push_back(0x00);
+ header.push_back(0x00);
+
+ // number of disk with start of central directory
+ header.push_back(0x00);
+ header.push_back(0x00);
+
+ // total central directory entries on this disk
+ header.push_back(BIT(central_dir_entries, 0, 8));
+ header.push_back(BIT(central_dir_entries, 8, 8));
+
+ // total central directory entries
+ header.push_back(BIT(central_dir_entries, 0, 8));
+ header.push_back(BIT(central_dir_entries, 8, 8));
+
+ // size of the central directory
+ header.push_back(BIT(central_dir_size, 0, 8));
+ header.push_back(BIT(central_dir_size, 8, 8));
+ header.push_back(BIT(central_dir_size, 16, 8));
+ header.push_back(BIT(central_dir_size, 24, 8));
+
+ // offset of central directory
+ header.push_back(BIT(central_dir_offset, 0, 8));
+ header.push_back(BIT(central_dir_offset, 8, 8));
+ header.push_back(BIT(central_dir_offset, 16, 8));
+ header.push_back(BIT(central_dir_offset, 24, 8));
+
+ // ZIP comment length
+ header.push_back(0x00);
+ header.push_back(0x00);
}
diff --git a/src/emu/save.h b/src/emu/save.h
index ad9b2eb6638..6c30d48e626 100644
--- a/src/emu/save.h
+++ b/src/emu/save.h
@@ -24,6 +24,8 @@
#include <type_traits>
#include <vector>
+#include <hashing.h>
+
//**************************************************************************
@@ -88,6 +90,7 @@ typedef named_delegate<void ()> save_prepost_delegate;
//**************************************************************************
class save_registered_item;
+class zlib_streamer;
class ram_state;
class rewinder;
@@ -104,7 +107,7 @@ class save_zip_state
public:
// the size threshold in bytes above which we will write an external file
- static constexpr u32 JSON_EXTERNAL_BINARY_THRESHOLD = 4096;
+ static constexpr u32 JSON_EXTERNAL_BINARY_THRESHOLD = 16 * 1024;
// construction
save_zip_state();
@@ -128,13 +131,10 @@ public:
save_zip_state &json_append_float(double value);
// stage an item to be output as raw data
- void add_data_file(char const *name, save_registered_item &item, void *base)
- {
- m_file_list.emplace_back(name, item, base);
- }
+ char const *add_data_file(char const *proposed_name, save_registered_item &item, uintptr_t base);
// commit the results to the given file
- void commit(FILE &output);
+ bool commit(emu_file &output);
private:
// check the reserve; if we're getting close, expand out one more chunk
@@ -147,18 +147,32 @@ private:
}
}
+ void create_end_of_central_directory(std::vector<u8> &header, u32 central_dir_entries, u64 central_dir_offset, u32 central_dir_size);
+ void create_zip_file_header(std::vector<u8> &local, std::vector<u8> &central, char const *filename, u64 local_offset);
+ void create_zip_file_footer(std::vector<u8> &local, std::vector<u8> &central, u32 filesize, u32 compressed, u32 crc);
+ bool write_data_recursive(zlib_streamer &zlib, save_registered_item &item, uintptr_t base);
+
// file_entry represents a single raw data file that will be written
struct file_entry
{
- file_entry(char const *name, save_registered_item &item, void *base) : m_item(item), m_name(name), m_base(base) { }
+ file_entry(char const *name, save_registered_item &item, uintptr_t base) :
+ m_item(item),
+ m_name(name),
+ m_base(base) { }
+
save_registered_item &m_item;
std::string m_name;
- void *m_base;
+ uintptr_t m_base;
+ std::vector<u8> m_central_directory;
};
// internal state
std::list<file_entry> m_file_list;
std::vector<char> m_json;
+ util::crc32_creator m_file_crc_accum;
+ u32 m_file_size_accum;
+ u16 m_archive_date;
+ u16 m_archive_time;
u32 m_json_reserved;
u32 m_json_offset;
};
@@ -199,9 +213,13 @@ public:
// simple getters
char const *name() const { return m_name.c_str(); }
save_type type() const { return m_type; }
+ bool is_struct_or_container() const { return (m_type == TYPE_STRUCT || m_type == TYPE_CONTAINER); }
+ bool is_array() const { return (m_type < TYPE_ARRAY); }
+ bool is_int() const { return (m_type == TYPE_INT || m_type == TYPE_UINT); }
+ bool is_int_or_float() const { return (is_int() || m_type == TYPE_FLOAT); }
std::list<save_registered_item> &subitems() { return m_items; }
uint32_t native_size() const { return m_native_size; }
- uint32_t count() const { return (m_type < TYPE_ARRAY) ? m_type : 1; }
+ uint32_t count() const { return is_array() ? m_type : 1; }
uintptr_t ptr_offset() const { return m_ptr_offset; }
// append a new item to the current one
@@ -504,9 +522,6 @@ public:
save_error save_file(emu_file &file);
save_error load_file(emu_file &file);
- // access to the root regist
- void test_dump();
-
private:
// state callback item
class state_callback