// license:BSD-3-Clause // copyright-holders:Aaron Giles /*************************************************************************** audit.cpp ROM set auditing functions. ***************************************************************************/ #include "emu.h" #include "audit.h" #include "sound/samples.h" #include "emuopts.h" #include "drivenum.h" #include "fileio.h" #include "romload.h" #include "softlist_dev.h" #include "chd.h" #include "path.h" #include //#define VERBOSE 1 #define LOG_OUTPUT_FUNC osd_printf_verbose #include "logmacro.h" namespace { struct parent_rom { parent_rom(device_type t, rom_entry const *r) : type(t), name(r->name()), hashes(r->hashdata()), length(rom_file_size(r)) { } std::reference_wrapper > type; std::string name; util::hash_collection hashes; uint64_t length; }; class parent_rom_vector : public std::vector { public: using std::vector::vector; void remove_redundant_parents(device_t const &device) { // remove parents with no shared ROMs const_reverse_iterator firstparent(crend()); for (rom_entry const *region = rom_first_region(device); region && ((crend() == firstparent) || (back().type.get() != firstparent->type.get())); region = rom_next_region(region)) { for (rom_entry const *rom = rom_first_file(region); rom; rom = rom_next_file(rom)) { util::hash_collection const hashes(rom->hashdata()); auto const match( std::find_if( crbegin(), firstparent, [rom, &hashes] (parent_rom const &r) { if (r.length != rom_file_size(rom)) return false; else if (!hashes.flag(util::hash_collection::FLAG_NO_DUMP)) return r.hashes == hashes; else return r.name == rom->name(); })); if (match != firstparent) { firstparent = std::find_if( crbegin(), match, [&match] (parent_rom const &r) { return r.type.get() == match->type.get(); }); if (back().type.get() == match->type.get()) break; } } } erase(firstparent.base(), cend()); while (!empty()) { // find where the next parent starts auto const last( std::find_if( std::next(cbegin()), cend(), [this] (parent_rom const &r) { return &front().type.get() != &r.type.get(); })); // examine dumped ROMs in this generation for (auto i = cbegin(); last != i; ++i) { if (!i->hashes.flag(util::hash_collection::FLAG_NO_DUMP)) { auto const match( std::find_if( last, cend(), [&i] (parent_rom const &r) { return (i->length == r.length) && (i->hashes == r.hashes); })); if (cend() == match) return; } } erase(cbegin(), last); } } std::add_pointer_t find_shared_device(device_t ¤t, std::string_view name, util::hash_collection const &hashes, uint64_t length) const { // if we're examining a child device, it will always have a perfect match if (current.owner()) return ¤t.type(); // scan backwards through parents for a matching definition bool const dumped(!hashes.flag(util::hash_collection::FLAG_NO_DUMP)); std::add_pointer_t best(nullptr); for (const_reverse_iterator it = crbegin(); crend() != it; ++it) { if (it->length == length) { if (dumped) { if (it->hashes == hashes) return &it->type.get(); } else if (it->name == name) { if (it->hashes.flag(util::hash_collection::FLAG_NO_DUMP)) return &it->type.get(); else if (!best) best = &it->type.get(); } } } return best; } std::pair, bool> actual_matches_shared(device_t ¤t, media_auditor::audit_record const &record) { // no result if no matching file was found if ((record.status() != media_auditor::audit_status::GOOD) && (record.status() != media_auditor::audit_status::FOUND_INVALID)) return std::make_pair(nullptr, false); // if we're examining a child device, scan it first bool matches_device_undumped(false); if (current.owner()) { for (const rom_entry *region = rom_first_region(current); region; region = rom_next_region(region)) { for (const rom_entry *rom = rom_first_file(region); rom; rom = rom_next_file(rom)) { if (rom_file_size(rom) == record.actual_length()) { util::hash_collection const hashes(rom->hashdata()); if (hashes == record.actual_hashes()) return std::make_pair(¤t.type(), empty()); else if (hashes.flag(util::hash_collection::FLAG_NO_DUMP) && (rom->name() == record.name())) matches_device_undumped = true; } } } } // look for a matching parent ROM std::add_pointer_t closest_bad(nullptr); for (const_reverse_iterator it = crbegin(); crend() != it; ++it) { if (it->length == record.actual_length()) { if (it->hashes == record.actual_hashes()) return std::make_pair(&it->type.get(), it->type.get() == front().type.get()); else if (it->hashes.flag(util::hash_collection::FLAG_NO_DUMP) && (it->name == record.name())) closest_bad = &it->type.get(); } } // fall back to the nearest bad dump if (closest_bad) return std::make_pair(closest_bad, front().type.get() == *closest_bad); else if (matches_device_undumped) return std::make_pair(¤t.type(), empty()); else return std::make_pair(nullptr, false); } }; } // anonymous namespace //************************************************************************** // CORE FUNCTIONS //************************************************************************** //------------------------------------------------- // media_auditor - constructor //------------------------------------------------- media_auditor::media_auditor(const driver_enumerator &enumerator) : m_enumerator(enumerator) , m_validation(AUDIT_VALIDATE_FULL) { } //------------------------------------------------- // audit_media - audit the media described by the // currently-enumerated driver //------------------------------------------------- media_auditor::summary media_auditor::audit_media(const char *validation) { // start fresh m_record_list.clear(); // store validation for later m_validation = validation; // first walk the parent chain for required ROMs parent_rom_vector parentroms; for (auto drvindex = m_enumerator.find(m_enumerator.driver().parent); 0 <= drvindex; drvindex = m_enumerator.find(m_enumerator.driver(drvindex).parent)) { game_driver const &parent(m_enumerator.driver(drvindex)); LOG("Checking parent %s for ROM files\n", parent.type.shortname()); std::vector const roms(rom_build_entries(parent.rom)); for (rom_entry const *region = rom_first_region(&roms.front()); region; region = rom_next_region(region)) { for (rom_entry const *rom = rom_first_file(region); rom; rom = rom_next_file(rom)) { LOG("Adding parent ROM %s\n", rom->name()); parentroms.emplace_back(parent.type, rom); } } } parentroms.remove_redundant_parents(m_enumerator.config()->root_device()); // count ROMs required/found std::size_t found(0); std::size_t required(0); std::size_t shared_found(0); std::size_t shared_required(0); std::size_t parent_found(0); // iterate over devices and regions std::vector searchpath; for (device_t &device : device_enumerator(m_enumerator.config()->root_device())) { searchpath.clear(); // now iterate over regions and ROMs within for (const rom_entry *region = rom_first_region(device); region; region = rom_next_region(region)) { for (const rom_entry *rom = rom_first_file(region); rom; rom = rom_next_file(rom)) { if (searchpath.empty()) { LOG("Audit media for device %s(%s)\n", device.shortname(), device.tag()); searchpath = device.searchpath(); } // look for a matching parent or device ROM std::string const &name(rom->name()); util::hash_collection const hashes(rom->hashdata()); bool const dumped(!hashes.flag(util::hash_collection::FLAG_NO_DUMP)); std::add_pointer_t const shared_device(parentroms.find_shared_device(device, name, hashes, rom_file_size(rom))); if (shared_device) LOG("File '%s' %s%sdumped shared with %s\n", name, ROM_ISOPTIONAL(rom) ? "optional " : "", dumped ? "" : "un", shared_device->shortname()); else LOG("File '%s' %s%sdumped\n", name, ROM_ISOPTIONAL(rom) ? "optional " : "", dumped ? "" : "un"); // count the number of files with hashes if (dumped && !ROM_ISOPTIONAL(rom)) { required++; if (shared_device) shared_required++; } audit_record *record(nullptr); if (ROMREGION_ISROMDATA(region)) record = &audit_one_rom(searchpath, rom); else if (ROMREGION_ISDISKDATA(region)) record = &audit_one_disk(rom, device); if (record) { // see if the actual content found belongs to a parent auto const matchesshared(parentroms.actual_matches_shared(device, *record)); if (matchesshared.first) LOG("Actual ROM file shared with %sparent %s\n", matchesshared.second ? "immediate " : "", matchesshared.first->shortname()); // count the number of files that are found. if ((record->status() == audit_status::GOOD) || ((record->status() == audit_status::FOUND_INVALID) && !matchesshared.first)) { found++; if (shared_device) shared_found++; if (matchesshared.second) parent_found++; } record->set_shared_device(shared_device); } } } if (!searchpath.empty()) LOG("Total required=%u (shared=%u) found=%u (shared=%u parent=%u)\n", required, shared_required, found, shared_found, parent_found); } // if we only find files that are in the parent and either the set has no unique files or the parent is not found, then assume we don't have the set at all if ((found == shared_found) && required && (found != required) && ((required != shared_required) || !parent_found)) { m_record_list.clear(); return NOTFOUND; } // return a summary return summarize(m_enumerator.driver().name); } //------------------------------------------------- // audit_device - audit the device //------------------------------------------------- media_auditor::summary media_auditor::audit_device(device_t &device, const char *validation) { // start fresh m_record_list.clear(); // store validation for later m_validation = validation; std::size_t found = 0; std::size_t required = 0; std::vector searchpath; audit_regions( [this, &device, &searchpath] (rom_entry const *region, rom_entry const *rom) -> audit_record const * { if (ROMREGION_ISROMDATA(region)) { if (searchpath.empty()) searchpath = device.searchpath(); return &audit_one_rom(searchpath, rom); } else if (ROMREGION_ISDISKDATA(region)) { return &audit_one_disk(rom, device); } else { return nullptr; } }, rom_first_region(device), found, required); if ((found == 0) && (required > 0)) { m_record_list.clear(); return NOTFOUND; } // return a summary return summarize(device.shortname()); } //------------------------------------------------- // audit_software //------------------------------------------------- media_auditor::summary media_auditor::audit_software(software_list_device &swlist, const software_info &swinfo, const char *validation) { // start fresh m_record_list.clear(); // store validation for later m_validation = validation; std::size_t found = 0; std::size_t required = 0; // now iterate over software parts std::vector searchpath; auto const do_audit = [this, &swlist, &swinfo, &searchpath] (rom_entry const *region, rom_entry const *rom) -> audit_record const * { if (ROMREGION_ISROMDATA(region)) { if (searchpath.empty()) searchpath = rom_load_manager::get_software_searchpath(swlist, swinfo); return &audit_one_rom(searchpath, rom); } else if (ROMREGION_ISDISKDATA(region)) { return &audit_one_disk(rom, swlist, swinfo); } else { return nullptr; } }; for (const software_part &part : swinfo.parts()) audit_regions(do_audit, part.romdata().data(), found, required); if ((found == 0) && (required > 0)) { m_record_list.clear(); return NOTFOUND; } // return a summary return summarize(swlist.list_name().c_str()); } //------------------------------------------------- // audit_samples - validate the samples for the // currently-enumerated driver //------------------------------------------------- media_auditor::summary media_auditor::audit_samples() { // start fresh m_record_list.clear(); std::size_t required = 0; std::size_t found = 0; // iterate over sample entries for (samples_device &device : samples_device_enumerator(m_enumerator.config()->root_device())) { // by default we just search using the driver name std::string searchpath(m_enumerator.driver().name); // add the alternate path if present samples_iterator iter(device); if (iter.altbasename() != nullptr) searchpath.append(";").append(iter.altbasename()); // iterate over samples in this entry for (const char *samplename = iter.first(); samplename; samplename = iter.next()) { required++; // create a new record audit_record &record = *m_record_list.emplace(m_record_list.end(), samplename, media_type::SAMPLE); // look for the files emu_file file(m_enumerator.options().sample_path(), OPEN_FLAG_READ | OPEN_FLAG_NO_PRELOAD); path_iterator path(searchpath); std::string curpath; while (path.next(curpath)) { util::path_append(curpath, samplename); // attempt to access the file (.flac) or (.wav) std::error_condition filerr = file.open(curpath + ".flac"); if (filerr) filerr = file.open(curpath + ".wav"); if (!filerr) { record.set_status(audit_status::GOOD, audit_substatus::GOOD); found++; } else { record.set_status(audit_status::NOT_FOUND, audit_substatus::NOT_FOUND); } } } } if ((found == 0) && (required > 0)) { m_record_list.clear(); return NOTFOUND; } // return a summary return summarize(m_enumerator.driver().name); } //------------------------------------------------- // summary - generate a summary, with an optional // string format //------------------------------------------------- media_auditor::summary media_auditor::summarize(const char *name, std::ostream *output) const { if (m_record_list.empty()) return NONE_NEEDED; // loop over records summary overall_status = CORRECT; for (audit_record const &record : m_record_list) { // skip anything that's fine if (record.substatus() == audit_substatus::GOOD) continue; // output the game name, file name, and length (if applicable) if (output) { if (name) util::stream_format(*output, "%-12s: %s", name, record.name()); else util::stream_format(*output, "%s", record.name()); if (record.expected_length() > 0) util::stream_format(*output, " (%d bytes)", record.expected_length()); *output << " - "; } // use the substatus for finer details summary best_new_status = INCORRECT; switch (record.substatus()) { case audit_substatus::GOOD_NEEDS_REDUMP: if (output) *output << "NEEDS REDUMP\n"; best_new_status = BEST_AVAILABLE; break; case audit_substatus::FOUND_NODUMP: if (output) *output << "NO GOOD DUMP KNOWN\n"; best_new_status = BEST_AVAILABLE; break; case audit_substatus::FOUND_BAD_CHECKSUM: if (output) { util::stream_format(*output, "INCORRECT CHECKSUM:\n"); util::stream_format(*output, "EXPECTED: %s\n", record.expected_hashes().macro_string()); util::stream_format(*output, " FOUND: %s\n", record.actual_hashes().macro_string()); } break; case audit_substatus::FOUND_WRONG_LENGTH: if (output) util::stream_format(*output, "INCORRECT LENGTH: %d bytes\n", record.actual_length()); break; case audit_substatus::NOT_FOUND: if (output) { std::add_pointer_t const shared_device = record.shared_device(); if (shared_device) util::stream_format(*output, "NOT FOUND (%s)\n", shared_device->shortname()); else util::stream_format(*output, "NOT FOUND\n"); } break; case audit_substatus::NOT_FOUND_NODUMP: if (output) *output << "NOT FOUND - NO GOOD DUMP KNOWN\n"; best_new_status = BEST_AVAILABLE; break; case audit_substatus::NOT_FOUND_OPTIONAL: if (output) *output << "NOT FOUND BUT OPTIONAL\n"; best_new_status = BEST_AVAILABLE; break; default: assert(false); } // downgrade the overall status if necessary overall_status = (std::max)(overall_status, best_new_status); } return overall_status; } //------------------------------------------------- // audit_regions - validate/count for regions //------------------------------------------------- template void media_auditor::audit_regions(T do_audit, const rom_entry *region, std::size_t &found, std::size_t &required) { // now iterate over regions std::vector searchpath; for ( ; region; region = rom_next_region(region)) { // now iterate over rom definitions for (rom_entry const *rom = rom_first_file(region); rom; rom = rom_next_file(rom)) { // count the number of files with hashes util::hash_collection const hashes(rom->hashdata()); if (!hashes.flag(util::hash_collection::FLAG_NO_DUMP) && !ROM_ISOPTIONAL(rom)) required++; audit_record const *const record = do_audit(region, rom); // count the number of files that are found. if (record && ((record->status() == audit_status::GOOD) || (record->status() == audit_status::FOUND_INVALID))) found++; } } } //------------------------------------------------- // audit_one_rom - validate a single ROM entry //------------------------------------------------- media_auditor::audit_record &media_auditor::audit_one_rom(const std::vector &searchpath, const rom_entry *rom) { // allocate and append a new record audit_record &record = *m_record_list.emplace(m_record_list.end(), *rom, media_type::ROM); // see if we have a CRC and extract it if so uint32_t crc = 0; bool const has_crc = record.expected_hashes().crc(crc); // find the file and checksum it, getting the file length along the way emu_file file(m_enumerator.options().media_path(), searchpath, OPEN_FLAG_READ | OPEN_FLAG_NO_PRELOAD); file.set_restrict_to_mediapath(1); // open the file if we can std::error_condition filerr; if (has_crc) filerr = file.open(record.name(), crc); else filerr = file.open(record.name()); // if it worked, get the actual length and hashes, then stop if (!filerr) record.set_actual(file.hashes(m_validation), file.size()); // compute the final status compute_status(record, rom, record.actual_length() != 0); return record; } //------------------------------------------------- // audit_one_disk - validate a single disk entry //------------------------------------------------- template media_auditor::audit_record &media_auditor::audit_one_disk(const rom_entry *rom, T &&... args) { // allocate and append a new record audit_record &record = *m_record_list.emplace(m_record_list.end(), *rom, media_type::DISK); // open the disk chd_file source; const std::error_condition err = rom_load_manager::open_disk_image(m_enumerator.options(), std::forward(args)..., rom, source); // FIXME: A CHD with an invalid header or missing parent is treated as not found. // We need a way to report more detailed errors for bad disk images. // if we succeeded, get the hashes if (!err) { util::hash_collection hashes; // if there's a SHA1 hash, add them to the output hash if (source.sha1() != util::sha1_t::null) hashes.add_sha1(source.sha1()); // update the actual values record.set_actual(hashes); } // compute the final status compute_status(record, rom, !err); return record; } //------------------------------------------------- // compute_status - compute a detailed status // based on the information we have //------------------------------------------------- void media_auditor::compute_status(audit_record &record, const rom_entry *rom, bool found) { // if not found, provide more details if (!found) { if (record.expected_hashes().flag(util::hash_collection::FLAG_NO_DUMP)) record.set_status(audit_status::NOT_FOUND, audit_substatus::NOT_FOUND_NODUMP); else if (ROM_ISOPTIONAL(rom)) record.set_status(audit_status::NOT_FOUND, audit_substatus::NOT_FOUND_OPTIONAL); else record.set_status(audit_status::NOT_FOUND, audit_substatus::NOT_FOUND); } else { if (record.expected_length() != record.actual_length()) record.set_status(audit_status::FOUND_INVALID, audit_substatus::FOUND_WRONG_LENGTH); else if (record.expected_hashes().flag(util::hash_collection::FLAG_NO_DUMP)) record.set_status(audit_status::GOOD, audit_substatus::FOUND_NODUMP); else if (record.expected_hashes() != record.actual_hashes()) record.set_status(audit_status::FOUND_INVALID, audit_substatus::FOUND_BAD_CHECKSUM); else if (record.expected_hashes().flag(util::hash_collection::FLAG_BAD_DUMP)) record.set_status(audit_status::GOOD, audit_substatus::GOOD_NEEDS_REDUMP); else record.set_status(audit_status::GOOD, audit_substatus::GOOD); } } //------------------------------------------------- // audit_record - constructor //------------------------------------------------- media_auditor::audit_record::audit_record(const rom_entry &media, media_type type) : m_type(type) , m_status(audit_status::UNVERIFIED) , m_substatus(audit_substatus::UNVERIFIED) , m_name(media.name()) , m_explength(rom_file_size(&media)) , m_length(0) , m_exphashes(media.hashdata()) , m_hashes() , m_shared_device(nullptr) { } media_auditor::audit_record::audit_record(const char *name, media_type type) : m_type(type) , m_status(audit_status::UNVERIFIED) , m_substatus(audit_substatus::UNVERIFIED) , m_name(name) , m_explength(0) , m_length(0) , m_exphashes() , m_hashes() , m_shared_device(nullptr) { }