diff options
author | 2019-10-29 10:49:22 +1100 | |
---|---|---|
committer | 2019-10-29 10:49:22 +1100 | |
commit | e4ee887d5cdffb6767e1078fe9dca32d7c920f2e (patch) | |
tree | 1c3ecc61329eae8cbad23cf1d2ca03a01764f739 /docs/release/scripts/minimaws/lib | |
parent | 1e8a03afab272a010763935b564c742155513797 (diff) |
0.215 Release filestag215
Diffstat (limited to 'docs/release/scripts/minimaws/lib')
-rw-r--r-- | docs/release/scripts/minimaws/lib/assets/machine.js | 8 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/assets/romident.js | 488 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/assets/style.css | 2 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/auxverbs.py | 155 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/dbaccess.py | 89 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/htmltmpl.py | 29 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/lxparse.py | 16 | ||||
-rw-r--r-- | docs/release/scripts/minimaws/lib/wsgiserve.py | 135 |
8 files changed, 892 insertions, 30 deletions
diff --git a/docs/release/scripts/minimaws/lib/assets/machine.js b/docs/release/scripts/minimaws/lib/assets/machine.js index d0774d02c0a..d40dca12be0 100644 --- a/docs/release/scripts/minimaws/lib/assets/machine.js +++ b/docs/release/scripts/minimaws/lib/assets/machine.js @@ -136,7 +136,7 @@ var fetch_bios_sets = (function () { pending[device] = true; var req = new XMLHttpRequest(); - req.open('GET', appurl + 'rpc/bios/' + device, true); + req.open('GET', appurl + 'rpc/bios/' + encodeURIComponent(device), true); req.responseType = 'json'; req.onload = function () @@ -171,7 +171,7 @@ var fetch_machine_flags = (function () { pending[device] = true; var req = new XMLHttpRequest(); - req.open('GET', appurl + 'rpc/flags/' + device, true); + req.open('GET', appurl + 'rpc/flags/' + encodeURIComponent(device), true); req.responseType = 'json'; req.onload = function () @@ -413,7 +413,7 @@ function make_slot_change_handler(name, slot, defaults, dfltbtn) row.appendChild(document.createElement('th')).textContent = 'Short name:'; var link = row.appendChild(document.createElement('td')).appendChild(document.createElement('a')); link.textContent = selection.device; - link.setAttribute('href', appurl + 'machine/' + selection.device); + link.setAttribute('href', appurl + 'machine/' + encodeURIComponent(selection.device)); // if we have emulation flags, populate now, otherwise fetch asynchronously if (!Object.prototype.hasOwnProperty.call(machine_flags, selection.device)) @@ -487,7 +487,7 @@ function fetch_slots(machine) function make_request(device) { var req = new XMLHttpRequest(); - req.open('GET', appurl + 'rpc/slots/' + device, true); + req.open('GET', appurl + 'rpc/slots/' + encodeURIComponent(device), true); req.responseType = 'json'; req.onload = function () diff --git a/docs/release/scripts/minimaws/lib/assets/romident.js b/docs/release/scripts/minimaws/lib/assets/romident.js new file mode 100644 index 00000000000..cbe1af89fdf --- /dev/null +++ b/docs/release/scripts/minimaws/lib/assets/romident.js @@ -0,0 +1,488 @@ +// license:BSD-3-Clause +// copyright-holders:Vas Crabb + +var matched_names = new Array(); +var unmatched_names = new Array(); +var dump_info = Object.create(null); +var machine_info = Object.create(null); +var crc_table = new Uint32Array(256); + +for (var i = 0; i < 256; i++) +{ + var crc = i; + for (var b = 0; b < 8; b++) + crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0x00000000); + crc_table[i] = crc; +} + + +function get_chd_sha1(header) +{ + if (header.byteLength < 16) + return null; + if (String.fromCharCode.apply(null, new Uint8Array(header, 0, 8)) != 'MComprHD') + return null; + var view = new DataView(header); + var headerlen = view.getUint32(8, false); + var version = view.getUint32(12, false); + var sha1offs; + if (version == 3) + { + if (headerlen != 120) + return null; + sha1offs = 80; + } + else if (version == 4) + { + if (headerlen != 108) + return null; + sha1offs = 48; + } + else if (version == 5) + { + if (headerlen != 124) + return null; + sha1offs = 84; + } + else + { + return null; + } + if (header.byteLength < (sha1offs + 20)) + return null; + return Array.from(new Uint8Array(header, sha1offs, 20)).map(x => x.toString(16).padStart(2, '0')).join(''); +} + + +function get_sha1_group(sha1) +{ + if (Object.hasOwnProperty.call(dump_info, sha1)) + { + return dump_info[sha1]; + } + else + { + var result = new Object(); + result.crc = Object.create(null); + dump_info[sha1] = result; + return result; + } +} + + +function add_matches(table, names, matches) +{ + for (var i = 0; i < names.length; i++) + { + var row = table.appendChild(document.createElement('tr')); + var name = row.appendChild(document.createElement('th')); + name.textContent = names[i]; + if (matches !== null) + { + name.setAttribute('rowspan', matches.length); + row.appendChild(document.createElement('td')).textContent = matches[0].name; + var bad = row.appendChild(document.createElement('td')); + if (matches[0].bad) + bad.textContent = 'BAD'; + for (var j = 1; j < matches.length; j++) + { + row = table.appendChild(document.createElement('tr')); + row.appendChild(document.createElement('td')).textContent = matches[j].name; + bad = row.appendChild(document.createElement('td')); + if (matches[j].bad) + bad.textContent = 'BAD'; + } + } + } +} + + +function add_unmatched(names, crc, sha1) +{ + var table; + if (unmatched_names.length > 0) + { + table = document.getElementById('table-unmatched'); + } + else + { + var div = document.getElementById('div-machines'); + var heading = document.body.insertBefore(document.createElement('h2'), div); + heading.textContent = 'Unmatched'; + table = document.body.insertBefore(document.createElement('table'), div); + table.setAttribute('id', 'table-unmatched'); + } + var row = table.appendChild(document.createElement('tr')); + var content; + if (crc === null) + { + content = 'SHA1(' + sha1 + ')'; + } + else + { + if (crc < 0) + crc = 0xffffffff + crc + 1; + content = 'CRC(' + crc.toString(16).padStart(8, '0') + ') SHA1(' + sha1 + ')'; + } + for (var i = 0; i < names.length; i++) + { + row.appendChild(document.createElement('th')).textContent = names[i]; + row.appendChild(document.createElement('td')).textContent = content; + unmatched_names.push(names[i]); + } +} + + +function get_machine_table(shortname, description) +{ + if (Object.hasOwnProperty.call(machine_info, shortname)) + { + return machine_info[shortname]; + } + else + { + var div = document.getElementById('div-machines'); + var heading = div.appendChild(document.createElement('h2')); + var link = heading.appendChild(document.createElement('a')); + link.textContent = description; + link.setAttribute('href', appurl + 'machine/' + encodeURIComponent(shortname)); + var table = div.appendChild(document.createElement('table')); + machine_info[shortname] = table; + add_matches(table, matched_names, null); + return table; + } +} + + +function request_dumps(name, group, crc, sha1, url, progress) +{ + var req = new XMLHttpRequest(); + req.responseType = 'json'; + req.onload = + function () + { + if (req.status == 200) + { + var machines = Object.create(null); + var matched = Object.keys(req.response); + if (matched.length > 0) + { + Object.keys(machine_info).forEach( + function (shortname) + { + var table = machine_info[shortname]; + if (Object.hasOwnProperty.call(req.response, shortname)) + { + machines[shortname] = req.response[shortname].matches; + add_matches(table, group.names, req.response[shortname].matches); + } + else + { + add_matches(table, group.names, null); + } + }); + matched.forEach( + function (shortname) + { + if (!Object.hasOwnProperty.call(machine_info, shortname)) + { + var info = req.response[shortname]; + var table = get_machine_table(shortname, info.description); + machines[shortname] = req.response[shortname].matches; + add_matches(table, group.names, info.matches); + } + }); + for (var i = 0; i < group.names.length; i++) + matched_names.push(group.names[i]); + } + else + { + add_unmatched(group.names, crc, sha1); + } + group.machines = machines; + progress.parentNode.removeChild(progress); + } + else + { + progress.textContent = 'Error identifying \u201C' + name + '\u201D: HTTP status code ' + req.status.toString(10); + if (crc !== null) + delete dump_info[sha1].crc[crc]; + else + delete dump_info[sha1].disk; + } + }; + req.onerror = + function () + { + progress.textContent = 'Error identifying \u201C' + name + '\u201D'; + if (crc !== null) + delete dump_info[sha1].crc[crc]; + else + delete dump_info[sha1].disk; + }; + req.onabort = + function () + { + progress.textContent = 'Error identifying \u201C' + name + '\u201D: server request aborted'; + if (crc !== null) + delete dump_info[sha1].crc[crc]; + else + delete dump_info[sha1].disk; + }; + req.open('GET', url); + req.send(); +} + + +function add_name(group, name, crc, sha1) +{ + if (group.names.indexOf(name) < 0) + { + group.names.push(name); + if (group.hasOwnProperty('machines')) + { + var machines = group.machines; + var names = [ name ]; + if (Object.keys(machines).length > 0) + { + Object.keys(machine_info).forEach( + function (shortname) + { + var table = machine_info[shortname]; + if (Object.hasOwnProperty.call(machines, shortname)) + add_matches(table, names, machines[shortname]); + else + add_matches(table, names, null); + }); + matched_names.push(name); + } + else + { + add_unmatched(names, crc, sha1); + } + } + } +} + + +function identify_file(file, trychd, progress) +{ + var digested = 0; + var crc = 0xffffffff; + var sha1st = new Uint32Array(5); + var sha1cnt = new Uint32Array(2); + var sha1buf = new DataView(new ArrayBuffer(64)); + + function rol(x, n) + { + return ((x << n) | (x >>> (32 - n))) & 0xffffffff; + } + + function sha1_process(data) + { + var d = new Uint32Array(sha1st); + + function b(i) + { + var r = data.getUint32(((i + 13) & 15) << 2, false); + r ^= data.getUint32(((i + 8) & 15) << 2, false); + r ^= data.getUint32(((i + 2) & 15) << 2, false); + r ^= data.getUint32((i & 15) << 2, false); + r = rol(r, 1); + data.setUint32((i & 15) << 2, r, false); + return r; + } + function r0(i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + ((d[(i + 3) % 5] & (d[(i + 2) % 5] ^ d[(i + 1) % 5])) ^ d[(i + 1) % 5]) + data.getUint32(i << 2, false) + 0x5a827999 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + function r1(i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + ((d[(i + 3) % 5] & (d[(i + 2) % 5] ^ d[(i + 1) % 5])) ^ d[(i + 1) % 5])+ b(i) + 0x5a827999 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + function r2(i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + (d[(i + 3) % 5] ^ d[(i + 2) % 5] ^ d[(i + 1) % 5]) + b(i) + 0x6ed9eba1 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + function r3(i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + (((d[(i + 3) % 5] | d[(i + 2) % 5]) & d[(i + 1) % 5]) | (d[(i + 3) % 5] & d[(i + 2) % 5])) + b(i) + 0x8f1bbcdc + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + function r4(i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + (d[(i + 3) % 5] ^ d[(i + 2) % 5] ^ d[(i + 1) % 5]) + b(i) + 0xca62c1d6 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + + var i = 0; + while (i < 16) + r0(i++); + while (i < 20) + r1(i++); + while (i < 40) + r2(i++); + while (i < 60) + r3(i++); + while (i < 80) + r4(i++); + for (i = 0; i < sha1st.length; i++) + sha1st[i] = (sha1st[i] + d[i]) & 0xffffffff; + } + + function sha1_digest(data, view) + { + var residual = sha1cnt[0]; + sha1cnt[0] = (sha1cnt[0] + (view.length << 3)) & 0xffffffff; + if (residual > sha1cnt[0]) + sha1cnt[1]++; + sha1cnt[1] += (view.length >>> 29); + residual = (residual >>> 3) & 63; + var offset = 0; + if ((residual + view.length) >= 64) + { + if (residual > 0) + { + for (offset = 0; (offset + residual) < 64; offset++) + sha1buf.setUint8(offset + residual, view[offset]); + sha1_process(sha1buf); + residual = 0; + } + for ( ; (view.length - offset) >= 64; offset += 64) + sha1_process(new DataView(data, offset, 64)); + } + for ( ; offset < view.length; residual++, offset++) + sha1buf.setUint8(residual, view[offset]); + } + + function sha1_finalise() + { + var lenbuf = new ArrayBuffer(8); + var lenview = new DataView(lenbuf); + lenview.setUint32(0, sha1cnt[1], false); + lenview.setUint32(4, sha1cnt[0], false); + var padbuf = new ArrayBuffer(64 - (63 & ((sha1cnt[0] >>> 3) + 8))); + var padview = new Uint8Array(padbuf); + padview[0] = 0x80; + for (var i = 1; i < padview.length; i++) + padview[i] = 0x00; + sha1_digest(padbuf, padview); + sha1_digest(lenbuf, new Uint8Array(lenbuf)); + var result = new Array(20); + for (var i = 0; i < 20; i++) + result[i] = (sha1st[4 - (i >>> 2)] >> ((3 - (i & 3)) << 3)) & 0x000000ff; + return result.map(x => x.toString(16).padStart(2, '0')).join(''); + } + + function process_chunk(e) + { + if (e.target.error === null) + { + var data = e.target.result; + if ((digested == 0) && trychd) + { + var chdsha1 = get_chd_sha1(data); + if (chdsha1 !== null) + { + var sha1grp = get_sha1_group(chdsha1); + if (!sha1grp.hasOwnProperty('disk')) + { + var diskgrp = new Object(); + diskgrp.names = [ file.name ]; + sha1grp.disk = diskgrp; + request_dumps(file.name, diskgrp, null, chdsha1, appurl + 'rpc/diskdumps?sha1=' + chdsha1, progress); + } + else + { + add_name(sha1grp.disk, file.name, crc, sha1); + progress.parentNode.removeChild(progress); + } + return; + } + } + var view = new Uint8Array(data); + for (var i = 0; i < view.length; i++) + crc = (crc >>> 8) ^ crc_table[(crc & 0x000000ff) ^ view[i]]; + sha1_digest(data, view); + digested += data.byteLength; + if (digested < file.size) + { + read_block(); + } + else + { + crc ^= 0xffffffff; + var sha1 = sha1_finalise(); + var sha1grp = get_sha1_group(sha1); + if (!Object.hasOwnProperty.call(sha1grp.crc, crc)) + { + var crcgrp = new Object(); + crcgrp.names = [ file.name ]; + sha1grp.crc[crc] = crcgrp; + var crcstr = ((crc < 0) ? (0xffffffff + crc + 1) : crc).toString(16).padStart(8, '0'); + request_dumps(file.name, crcgrp, crc, sha1, appurl + 'rpc/romdumps?crc=' + crcstr + '&sha1=' + sha1, progress); + } + else + { + add_name(sha1grp.crc[crc], file.name, crc, sha1); + progress.parentNode.removeChild(progress); + } + } + } + else + { + progress.textContent = 'Error identifying \u201C' + file.name + '\u201D: ' + e.target.error; + } + } + + function read_block() + { + var remaining = file.size - digested; + var chunk = file.slice(digested, digested + Math.min(remaining, 65536)); + var reader = new FileReader(); + reader.onload = process_chunk; + reader.readAsArrayBuffer(chunk); + }; + + sha1st[0] = 0xc3d2e1f0; + sha1st[1] = 0x10325476; + sha1st[2] = 0x98badcfe; + sha1st[3] = 0xefcdab89; + sha1st[4] = 0x67452301; + sha1cnt[0] = 0; + sha1cnt[1] = 0; + progress.textContent = 'Identifying \u201C' + file.name + '\u201D\u2026'; + read_block(); +} + + +function add_dump_files(files) +{ + var prog = document.getElementById('div-progress'); + for (var i = 0; i < files.length; i++) + { + var name = files[i].name; + var dot = name.lastIndexOf('.'); + var trychd = (dot > 0) && (name.substring(dot + 1).toLowerCase() == 'chd'); + identify_file(files[i], trychd, prog.appendChild(document.createElement('p'))); + } +} + + +function div_dropzone_dragover(e) +{ + e.stopPropagation(); + e.preventDefault(); + e.dataTransfer.dropEffect = 'link'; +} + + +function div_dropzone_drop(e) +{ + e.stopPropagation(); + e.preventDefault(); + add_dump_files(e.dataTransfer.files); +} diff --git a/docs/release/scripts/minimaws/lib/assets/style.css b/docs/release/scripts/minimaws/lib/assets/style.css index 75ea0fa93de..1e6ebd57ec0 100644 --- a/docs/release/scripts/minimaws/lib/assets/style.css +++ b/docs/release/scripts/minimaws/lib/assets/style.css @@ -7,5 +7,7 @@ td { padding-left: 0.25em; padding-right: 0.25em } table[class=sysinfo] th { text-align: right } +div[class=dropzone] { padding: 0 1em; border-width: 2px; border-radius: 1em; border-style: dashed } + dl[id=list-slot-options] dt { font-weight: bold; margin-top: 1em } dl[id=list-slot-options] dd table { margin-top: 0.5em; margin-bottom: 1em } diff --git a/docs/release/scripts/minimaws/lib/auxverbs.py b/docs/release/scripts/minimaws/lib/auxverbs.py index c68c528af6f..b67a2fafb99 100644 --- a/docs/release/scripts/minimaws/lib/auxverbs.py +++ b/docs/release/scripts/minimaws/lib/auxverbs.py @@ -5,7 +5,150 @@ from . import dbaccess +import codecs +import hashlib +import os +import os.path +import struct import sys +import zlib + + +class _Identifier(object): + def __init__(self, dbcurs, **kwargs): + super(_Identifier, self).__init__(**kwargs) + self.dbcurs = dbcurs + self.pathwidth = 0 + self.labelwidth = 0 + self.matches = { } + self.unmatched = [ ] + + def processRomFile(self, path, f): + crc, sha1 = self.digestRom(f) + matched = False + for shortname, description, label, bad in self.dbcurs.get_rom_dumps(crc, sha1): + matched = True + self.labelwidth = max(len(label), self.labelwidth) + romset = self.matches.get(shortname) + if romset is None: + romset = (description, []) + self.matches[shortname] = romset + romset[1].append((path, label, bad)) + if not matched: + self.unmatched.append((path, crc, sha1)) + self.pathwidth = max(len(path), self.pathwidth) + + def processChd(self, path, sha1): + matched = False + for shortname, description, label, bad in self.dbcurs.get_disk_dumps(sha1): + matched = True + self.labelwidth = max(len(label), self.labelwidth) + romset = self.matches.get(shortname) + if romset is None: + romset = (description, []) + self.matches[shortname] = romset + romset[1].append((path, label, bad)) + if not matched: + self.unmatched.append((path, None, sha1)) + self.pathwidth = max(len(path), self.pathwidth) + + def processFile(self, path): + if os.path.splitext(path)[1].lower() != '.chd': + with open(path, mode='rb', buffering=0) as f: + self.processRomFile(path, f) + else: + with open(path, mode='rb') as f: + sha1 = self.probeChd(f) + if sha1 is None: + f.seek(0) + self.processRomFile(path, f) + else: + self.processChd(path, sha1) + + def processPath(self, path, depth=0): + try: + if not os.path.isdir(path): + self.processFile(path) + elif depth > 5: + sys.stderr.write('Not examining \'%s\' - maximum depth exceeded\n') + else: + for name in os.listdir(path): + self.processPath(os.path.join(path, name), depth + 1) + except BaseException as e: + sys.stderr.write('Error identifying \'%s\': %s\n' % (path, e)) + + def printResults(self): + pw = self.pathwidth - (self.pathwidth % 4) + 4 + lw = self.labelwidth - (self.labelwidth % 4) + 4 + first = True + for shortname, romset in sorted(self.matches.items()): + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('%-20s%s\n' % (shortname, romset[0])) + for path, label, bad in romset[1]: + if bad: + sys.stdout.write(' %-*s= %-*s(BAD)\n' % (pw, path, lw, label)) + else: + sys.stdout.write(' %-*s= %s\n' % (pw, path, label)) + if self.unmatched: + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('Unmatched\n') + for path, crc, sha1 in self.unmatched: + if crc is not None: + sys.stdout.write(' %-*sCRC(%08x) SHA1(%s)\n' % (pw, path, crc, sha1)) + else: + sys.stdout.write(' %-*sSHA1(%s)\n' % (pw, path, sha1)) + + @staticmethod + def iterateBlocks(f, s=65536): + while True: + buf = f.read(s) + if buf: + yield buf + else: + break + + @staticmethod + def digestRom(f): + crc = zlib.crc32(bytes()) + sha = hashlib.sha1() + for block in _Identifier.iterateBlocks(f): + crc = zlib.crc32(block, crc) + sha.update(block) + return crc & 0xffffffff, sha.hexdigest() + + @staticmethod + def probeChd(f): + buf = f.read(16) + if (len(buf) != 16) or (buf[:8] != b'MComprHD'): + return None + headerlen, version = struct.unpack('>II', buf[8:]) + if version == 3: + if headerlen != 120: + return None + sha1offs = 80 + elif version == 4: + if headerlen != 108: + return None + sha1offs = 48 + elif version == 5: + if headerlen != 124: + return None + sha1offs = 84 + else: + return None + f.seek(sha1offs) + if f.tell() != sha1offs: + return None + buf = f.read(20) + if len(buf) != 20: + return None + return codecs.getencoder('hex_codec')(buf)[0].decode('ascii') def do_listfull(options): @@ -68,6 +211,7 @@ def do_listbrothers(options): dbcurs.close() dbconn.close() + def do_listaffected(options): dbconn = dbaccess.QueryConnection(options.database) dbcurs = dbconn.cursor() @@ -81,3 +225,14 @@ def do_listaffected(options): sys.stderr.write('No matching systems found for \'%s\'\n' % (options.pattern, )) dbcurs.close() dbconn.close() + + +def do_romident(options): + dbconn = dbaccess.QueryConnection(options.database) + dbcurs = dbconn.cursor() + ident = _Identifier(dbcurs) + for path in options.path: + ident.processPath(path) + ident.printResults() + dbcurs.close() + dbconn.close() diff --git a/docs/release/scripts/minimaws/lib/dbaccess.py b/docs/release/scripts/minimaws/lib/dbaccess.py index 57de6aeb236..24dbb163171 100644 --- a/docs/release/scripts/minimaws/lib/dbaccess.py +++ b/docs/release/scripts/minimaws/lib/dbaccess.py @@ -141,6 +141,35 @@ class SchemaQueries(object): ' size INTEGER NOT NULL,\n' \ ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ ' FOREIGN KEY (machine, size) REFERENCES ramoption (machine, size))' + CREATE_ROM = \ + 'CREATE TABLE rom (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' crc INTEGER NOT NULL,\n' \ + ' sha1 TEXT NOT NULL,\n' \ + ' UNIQUE (crc ASC, sha1 ASC))' + CREATE_ROMDUMP = \ + 'CREATE TABLE romdump (\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' rom INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (rom) REFERENCES rom (id),\n' \ + ' UNIQUE (machine, rom, name))' + CREATE_DISK = \ + 'CREATE TABLE disk (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' sha1 TEXT NOT NULL,\n' \ + ' UNIQUE (sha1 ASC))' + CREATE_DISKDUMP = \ + 'CREATE TABLE diskdump (\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' disk INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (disk) REFERENCES disk (id),\n' \ + ' UNIQUE (machine, disk, name))' CREATE_TEMPORARY_DEVICEREFERENCE = 'CREATE TEMPORARY TABLE temp_devicereference (id INTEGER PRIMARY KEY, machine INTEGER NOT NULL, device TEXT NOT NULL, UNIQUE (machine, device))' CREATE_TEMPORARY_SLOTOPTION = 'CREATE TEMPORARY TABLE temp_slotoption (id INTEGER PRIMARY KEY, slot INTEGER NOT NULL, device TEXT NOT NULL, name TEXT NOT NULL)' @@ -164,6 +193,10 @@ class SchemaQueries(object): INDEX_DIPSWITCH_MACHINE_ISCONFIG = 'CREATE INDEX dipswitch_machine_isconfig ON dipswitch (machine ASC, isconfig ASC)' + INDEX_ROMDUMP_ROM = 'CREATE INDEX romdump_rom ON romdump (rom ASC)' + + INDEX_DISKDUMP_DISK = 'CREATE INDEX diskdump_disk ON diskdump (disk ASC)' + DROP_MACHINE_ISDEVICE_SHORTNAME = 'DROP INDEX IF EXISTS machine_isdevice_shortname' DROP_MACHINE_ISDEVICE_DESCRIPTION = 'DROP INDEX IF EXISTS machine_isdevice_description' DROP_MACHINE_RUNNABLE_SHORTNAME = 'DROP INDEX IF EXISTS machine_runnable_shortname' @@ -178,6 +211,10 @@ class SchemaQueries(object): DROP_DIPSWITCH_MACHINE_ISCONFIG = 'DROP INDEX IF EXISTS dipswitch_machine_isconfig' + DROP_ROMDUMP_ROM = 'DROP INDEX IF EXISTS romdump_rom' + + DROP_DISKDUMP_DISK = 'DROP INDEX IF EXISTS diskdump_disk' + CREATE_TABLES = ( CREATE_FEATURETYPE, CREATE_SOURCEFILE, @@ -196,7 +233,11 @@ class SchemaQueries(object): CREATE_SLOTOPTION, CREATE_SLOTDEFAULT, CREATE_RAMOPTION, - CREATE_RAMDEFAULT) + CREATE_RAMDEFAULT, + CREATE_ROM, + CREATE_ROMDUMP, + CREATE_DISK, + CREATE_DISKDUMP) CREATE_TEMPORARY_TABLES = ( CREATE_TEMPORARY_DEVICEREFERENCE, @@ -212,7 +253,9 @@ class SchemaQueries(object): INDEX_SYSTEM_MANUFACTURER, INDEX_ROMOF_PARENT, INDEX_CLONEOF_PARENT, - INDEX_DIPSWITCH_MACHINE_ISCONFIG) + INDEX_DIPSWITCH_MACHINE_ISCONFIG, + INDEX_ROMDUMP_ROM, + INDEX_DISKDUMP_DISK) DROP_INDEXES = ( DROP_MACHINE_ISDEVICE_SHORTNAME, @@ -223,7 +266,9 @@ class SchemaQueries(object): DROP_SYSTEM_MANUFACTURER, DROP_ROMOF_PARENT, DROP_CLONEOF_PARENT, - DROP_DIPSWITCH_MACHINE_ISCONFIG) + DROP_DIPSWITCH_MACHINE_ISCONFIG, + DROP_ROMDUMP_ROM, + DROP_DISKDUMP_DISK) class UpdateQueries(object): @@ -242,6 +287,10 @@ class UpdateQueries(object): ADD_SLOT = 'INSERT INTO slot (machine, name) VALUES (?, ?)' ADD_RAMOPTION = 'INSERT INTO ramoption (machine, size, name) VALUES (?, ?, ?)' ADD_RAMDEFAULT = 'INSERT INTO ramdefault (machine, size) VALUES (?, ?)' + ADD_ROM = 'INSERT OR IGNORE INTO rom (crc, sha1) VALUES (?, ?)' + ADD_ROMDUMP = 'INSERT OR IGNORE INTO romdump (machine, rom, name, bad) SELECT ?, id, ?, ? FROM rom WHERE crc = ? AND sha1 = ?' + ADD_DISK = 'INSERT OR IGNORE INTO disk (sha1) VALUES (?)' + ADD_DISKDUMP = 'INSERT OR IGNORE INTO diskdump (machine, disk, name, bad) SELECT ?, id, ?, ? FROM disk WHERE sha1 = ?' ADD_TEMPORARY_DEVICEREFERENCE = 'INSERT OR IGNORE INTO temp_devicereference (machine, device) VALUES (?, ?)' ADD_TEMPORARY_SLOTOPTION = 'INSERT INTO temp_slotoption (slot, device, name) VALUES (?, ?, ?)' @@ -458,6 +507,20 @@ class QueryCursor(object): 'ORDER BY ramoption.size', (machine, )) + def get_rom_dumps(self, crc, sha1): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, romdump.name AS label, romdump.bad AS bad ' \ + 'FROM romdump LEFT JOIN machine ON romdump.machine = machine.id ' \ + 'WHERE romdump.rom = (SELECT id FROM rom WHERE crc = ? AND sha1 = ?)', + (crc, sha1)) + + def get_disk_dumps(self, sha1): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, diskdump.name AS label, diskdump.bad AS bad ' \ + 'FROM diskdump LEFT JOIN machine ON diskdump.machine = machine.id ' \ + 'WHERE diskdump.disk = (SELECT id FROM disk WHERE sha1 = ?)', + (sha1, )) + class UpdateCursor(object): def __init__(self, dbconn, **kwargs): @@ -536,14 +599,30 @@ class UpdateCursor(object): self.dbcurs.execute(UpdateQueries.ADD_RAMDEFAULT, (machine, size)) return self.dbcurs.lastrowid + def add_rom(self, crc, sha1): + self.dbcurs.execute(UpdateQueries.ADD_ROM, (crc, sha1)) + return self.dbcurs.lastrowid + + def add_romdump(self, machine, name, crc, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_ROMDUMP, (machine, name, 1 if bad else 0, crc, sha1)) + return self.dbcurs.lastrowid + + def add_disk(self, sha1): + self.dbcurs.execute(UpdateQueries.ADD_DISK, (sha1, )) + return self.dbcurs.lastrowid + + def add_diskdump(self, machine, name, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_DISKDUMP, (machine, name, 1 if bad else 0, sha1)) + return self.dbcurs.lastrowid + class QueryConnection(object): def __init__(self, database, **kwargs): super(QueryConnection, self).__init__(**kwargs) if sys.version_info >= (3, 4): - self.dbconn = sqlite3.connect('file:' + urllib.request.pathname2url(database) + '?mode=ro', uri=True) + self.dbconn = sqlite3.connect('file:' + urllib.request.pathname2url(database) + '?mode=ro', uri=True, check_same_thread=False) else: - self.dbconn = sqlite3.connect(database) + self.dbconn = sqlite3.connect(database, check_same_thread=False) self.dbconn.row_factory = sqlite3.Row self.dbconn.execute('PRAGMA foreign_keys = ON') diff --git a/docs/release/scripts/minimaws/lib/htmltmpl.py b/docs/release/scripts/minimaws/lib/htmltmpl.py index 1a030d04e93..55034b0d079 100644 --- a/docs/release/scripts/minimaws/lib/htmltmpl.py +++ b/docs/release/scripts/minimaws/lib/htmltmpl.py @@ -163,3 +163,32 @@ SOURCEFILE_LIST_ROW = string.Template( ' <td>${sourcefile}</td>\n' \ ' <td style="text-align: right">${machines}</td>\n' \ ' </tr>\n') + +ROMIDENT_PAGE = string.Template( + '<!DOCTYPE html>\n' \ + '<html>\n' \ + '<head>\n' \ + ' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n' \ + ' <meta http-equiv="Content-Style-Type" content="text/css">\n' \ + ' <meta http-equiv="Content-Script-Type" content="text/javascript">\n' \ + ' <link rel="stylesheet" type="text/css" href="${assets}/style.css">\n' \ + ' <script type="text/javascript">\n' \ + ' var appurl="${app}"\n' \ + ' var assetsurl="${assets}"\n' \ + ' </script>\n' \ + ' <script type="text/javascript" src="${assets}/common.js"></script>\n' \ + ' <script type="text/javascript" src="${assets}/romident.js"></script>\n' \ + ' <title>Identify ROM/Disk Dumps</title>\n' \ + '</head>\n' \ + '<body>\n' \ + '<h1>Identify ROM/Disk Dumps</h1>\n' \ + '<p>No files are uploaded. Files are examined locally and checksums/digests are sent to the server. File checksums and digests may be logged on the server.</p>\n' \ + '<div id="div-dropzone" class="dropzone" ondragover="div_dropzone_dragover(event)" ondrop="div_dropzone_drop(event)">\n' \ + '<p><button type="button" onclick="document.getElementById("input-dumps").click()">Select ROM/disk dumps</button></p>\n' \ + '<p>Drag and drop ROM/disk dump files here to identify them.</p>\n' \ + '</div>\n' \ + '<input id="input-dumps" type="file" multiple onchange="add_dump_files(this.files)" style="display: none">\n' \ + '<div id="div-progress"></div>\n' \ + '<div id="div-machines"></div>\n' \ + '</body>\n' \ + '</html>\n') diff --git a/docs/release/scripts/minimaws/lib/lxparse.py b/docs/release/scripts/minimaws/lib/lxparse.py index dfd8cbae45e..96669ea99eb 100644 --- a/docs/release/scripts/minimaws/lib/lxparse.py +++ b/docs/release/scripts/minimaws/lib/lxparse.py @@ -206,6 +206,22 @@ class MachineHandler(ElementHandler): status = 0 if 'status' not in attrs else 2 if attrs['status'] == 'unemulated' else 1 overall = status if 'overall' not in attrs else 2 if attrs['overall'] == 'unemulated' else 1 self.dbcurs.add_feature(self.id, attrs['type'], status, overall) + elif name == 'rom': + crc = attrs.get('crc') + sha1 = attrs.get('sha1') + if (crc is not None) and (sha1 is not None): + crc = int(crc, 16) + sha1 = sha1.lower() + self.dbcurs.add_rom(crc, sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_romdump(self.id, attrs['name'], crc, sha1, status != 'good') + elif name == 'disk': + sha1 = attrs.get('sha1') + if sha1 is not None: + sha1 = sha1.lower() + self.dbcurs.add_disk(sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_diskdump(self.id, attrs['name'], sha1, status != 'good') self.setChildHandler(name, attrs, self.IGNORE) def endChildHandler(self, name, handler): diff --git a/docs/release/scripts/minimaws/lib/wsgiserve.py b/docs/release/scripts/minimaws/lib/wsgiserve.py index b30828e01e0..da9cbed0f36 100644 --- a/docs/release/scripts/minimaws/lib/wsgiserve.py +++ b/docs/release/scripts/minimaws/lib/wsgiserve.py @@ -13,13 +13,15 @@ import mimetypes import os.path import re import sys -import wsgiref.simple_server +import urllib import wsgiref.util if sys.version_info >= (3, ): import urllib.parse as urlparse + urlquote = urlparse.quote else: import urlparse + urlquote = urllib.quote class HandlerBase(object): @@ -96,10 +98,10 @@ class QueryPageHandler(HandlerBase): self.dbcurs = app.dbconn.cursor() def machine_href(self, shortname): - return cgi.escape(urlparse.urljoin(self.application_uri, 'machine/%s' % (shortname, )), True) + return cgi.escape(urlparse.urljoin(self.application_uri, 'machine/%s' % (urlquote(shortname), )), True) def sourcefile_href(self, sourcefile): - return cgi.escape(urlparse.urljoin(self.application_uri, 'sourcefile/%s' % (sourcefile, )), True) + return cgi.escape(urlparse.urljoin(self.application_uri, 'sourcefile/%s' % (urlquote(sourcefile), )), True) class MachineRpcHandlerBase(QueryPageHandler): @@ -174,21 +176,21 @@ class MachineHandler(QueryPageHandler): if parent: yield ( ' <tr><th>Parent Machine:</th><td><a href="%s">%s (%s)</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['cloneof']), True), cgi.escape(parent[1]), cgi.escape(machine_info['cloneof']))).encode('utf-8') + (self.machine_href(machine_info['cloneof']), cgi.escape(parent[1]), cgi.escape(machine_info['cloneof']))).encode('utf-8') else: yield ( ' <tr><th>Parent Machine:</th><td><a href="%s">%s</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['cloneof']), True), cgi.escape(machine_info['cloneof']))).encode('utf-8') + (self.machine_href(machine_info['cloneof']), cgi.escape(machine_info['cloneof']))).encode('utf-8') if (machine_info['romof'] is not None) and (machine_info['romof'] != machine_info['cloneof']): parent = self.dbcurs.listfull(machine_info['romof']).fetchone() if parent: yield ( ' <tr><th>Parent ROM set:</th><td><a href="%s">%s (%s)</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['romof']), True), cgi.escape(parent[1]), cgi.escape(machine_info['romof']))).encode('utf-8') + (self.machine_href(machine_info['romof']), cgi.escape(parent[1]), cgi.escape(machine_info['romof']))).encode('utf-8') else: yield ( ' <tr><th>Parent Machine:</th><td><a href="%s">%s</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['romof']), True), cgi.escape(machine_info['romof']))).encode('utf-8') + (self.machine_href(machine_info['romof']), cgi.escape(machine_info['romof']))).encode('utf-8') unemulated = [] imperfect = [] for feature, status, overall in self.dbcurs.get_feature_flags(id): @@ -404,10 +406,10 @@ class SourceFileHandler(QueryPageHandler): uri = urlparse.urljoin(self.application_uri, 'sourcefile') title = '' for part in parts: - uri = urlparse.urljoin(uri + '/', part) + uri = urlparse.urljoin(uri + '/', urlquote(part)) title += '<a href="{0}">{1}</a>/'.format(cgi.escape(uri, True), cgi.escape(part)) if linkfinal: - uri = urlparse.urljoin(uri + '/', final) + uri = urlparse.urljoin(uri + '/', urlquote(final)) return title + '<a href="{0}">{1}</a>'.format(cgi.escape(uri, True), cgi.escape(final)) else: return title + final @@ -424,6 +426,28 @@ class SourceFileHandler(QueryPageHandler): parent=cgi.escape(machine_info['cloneof'] or '')).encode('utf-8') +class RomIdentHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super(QueryPageHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + self.dbcurs = app.dbconn.cursor() + + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + self.start_response('200 OK', [('Content-type', 'text/html; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.form_page() + + def form_page(self): + yield htmltmpl.ROMIDENT_PAGE.substitute( + app=self.js_escape(cgi.escape(self.application_uri, True)), + assets=self.js_escape(cgi.escape(urlparse.urljoin(self.application_uri, 'static'), True))).encode('utf-8') + + class BiosRpcHandler(MachineRpcHandlerBase): def data_page(self, machine): result = { } @@ -477,12 +501,86 @@ class SlotsRpcHandler(MachineRpcHandlerBase): yield json.dumps(result).encode('utf-8') +class RomDumpsRpcHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super(RomDumpsRpcHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + try: + args = urlparse.parse_qs(self.environ['QUERY_STRING'], keep_blank_values=True, strict_parsing=True) + crc = args.get('crc') + sha1 = args.get('sha1') + if (len(args) == 2) and (crc is not None) and (len(crc) == 1) and (sha1 is not None) and (len(sha1) == 1): + crc = int(crc[0], 16) + sha1 = sha1[0] + self.start_response('200 OK', [('Content-type', 'application/json; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.data_page(crc, sha1) + except BaseException as e: + pass + self.start_response('500 %s' % (self.STATUS_MESSAGE[500], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(500) + + def data_page(self, crc, sha1): + result = { } + for shortname, description, label, bad in self.dbcurs.get_rom_dumps(crc, sha1): + machine = result.get(shortname) + if machine is None: + machine = { 'description': description, 'matches': [ ] } + result[shortname] = machine + machine['matches'].append({ 'name': label, 'bad': bool(bad) }) + yield json.dumps(result).encode('utf-8') + + +class DiskDumpsRpcHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super(DiskDumpsRpcHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + try: + args = urlparse.parse_qs(self.environ['QUERY_STRING'], keep_blank_values=True, strict_parsing=True) + sha1 = args.get('sha1') + if (len(args) == 1) and (sha1 is not None) and (len(sha1) == 1): + sha1 = sha1[0] + self.start_response('200 OK', [('Content-type', 'application/json; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.data_page(sha1) + except BaseException as e: + pass + self.start_response('500 %s' % (self.STATUS_MESSAGE[500], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(500) + + def data_page(self, sha1): + result = { } + for shortname, description, label, bad in self.dbcurs.get_disk_dumps(sha1): + machine = result.get(shortname) + if machine is None: + machine = { 'description': description, 'matches': [ ] } + result[shortname] = machine + machine['matches'].append({ 'name': label, 'bad': bool(bad) }) + yield json.dumps(result).encode('utf-8') + + class MiniMawsApp(object): JS_ESCAPE = re.compile('([\"\'\\\\])') RPC_SERVICES = { - 'bios': BiosRpcHandler, - 'flags': FlagsRpcHandler, - 'slots': SlotsRpcHandler } + 'bios': BiosRpcHandler, + 'flags': FlagsRpcHandler, + 'slots': SlotsRpcHandler, + 'romdumps': RomDumpsRpcHandler, + 'diskdumps': DiskDumpsRpcHandler } def __init__(self, dbfile, **kwargs): super(MiniMawsApp, self).__init__(**kwargs) @@ -493,11 +591,15 @@ class MiniMawsApp(object): def __call__(self, environ, start_response): application_uri = wsgiref.util.application_uri(environ) + if application_uri[-1] != '/': + application_uri += '/' module = wsgiref.util.shift_path_info(environ) if module == 'machine': return MachineHandler(self, application_uri, environ, start_response) elif module == 'sourcefile': return SourceFileHandler(self, application_uri, environ, start_response) + elif module == 'romident': + return RomIdentHandler(self, application_uri, environ, start_response) elif module == 'static': return AssetHandler(self.assetsdir, self, application_uri, environ, start_response) elif module == 'rpc': @@ -515,12 +617,3 @@ class MiniMawsApp(object): def js_escape(self, str): return self.JS_ESCAPE.sub('\\\\\\1', str).replace('\0', '\\0') - - -def run_server(options): - application = MiniMawsApp(options.database) - server = wsgiref.simple_server.make_server(options.host, options.port, application) - try: - server.serve_forever() - except KeyboardInterrupt: - pass |