diff options
Diffstat (limited to 'scripts/minimaws')
21 files changed, 3055 insertions, 234 deletions
diff --git a/scripts/minimaws/lib/__init__.py b/scripts/minimaws/lib/__init__.py index d89cb7f81ba..a8a22f4dbfe 100644 --- a/scripts/minimaws/lib/__init__.py +++ b/scripts/minimaws/lib/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb diff --git a/scripts/minimaws/lib/assets/common.js b/scripts/minimaws/lib/assets/common.js index 79200217ddb..3b10b785fdd 100644 --- a/scripts/minimaws/lib/assets/common.js +++ b/scripts/minimaws/lib/assets/common.js @@ -1,55 +1,127 @@ // license:BSD-3-Clause // copyright-holders:Vas Crabb +'use strict'; -function sort_table(tbl, col, dir, numeric) +function make_collapsible(heading, section) { - var tbody = tbl.tBodies[0]; - var trows = Array.prototype.slice.call(tbody.rows, 0).sort( - function (x, y) + var hidden = false; + var display = section.style.display; + var icon = heading.insertBefore(document.createElement('img'), heading.firstChild); + icon.setAttribute('src', assetsurl + '/disclosedown.svg'); + icon.setAttribute('class', 'disclosure'); + icon.addEventListener( + 'click', + function (event) { - if (numeric) - return dir * (parseInt(x.cells[col].textContent) - parseInt(y.cells[col].textContent)); + hidden = !hidden; + if (hidden) + { + icon.setAttribute('src', assetsurl + '/discloseup.svg'); + section.style.display = 'none'; + } else - return dir * x.cells[col].textContent.localeCompare(y.cells[col].textContent); - }) - trows.forEach(function (row) { tbody.appendChild(row); }); + { + icon.setAttribute('src', assetsurl + '/disclosedown.svg'); + section.style.display = display; + var headingtop = 0; + for (var element = heading; element !== null; element = element.offsetParent) + headingtop += element.offsetTop; + var sectionbot = section.offsetHeight; + for (var element = section; element !== null; element = element.offsetParent) + sectionbot += element.offsetTop; + if ((window.pageYOffset + window.innerHeight) < sectionbot) + { + if ((sectionbot - headingtop) > window.innerHeight) + heading.scrollIntoView(true); + else + window.scroll(window.pageXOffset, sectionbot - window.innerHeight); + } + } + }); + + return function () + { + hidden = true; + icon.setAttribute('src', assetsurl + '/discloseup.svg'); + section.style.display = 'none'; + }; } function make_table_sortable(tbl) { - var headers = tbl.tHead.rows[0].cells; - for (var i = 0; i < headers.length; i++) + var sorticons = new Array(tbl.tHead.rows[0].cells.length); + + function TableSortHelper(i) { - (function (col) - { - var dir = 1; - var sorticon = document.createElement('img'); - sorticon.setAttribute('src', assetsurl + '/sortind.png'); - sorticon.style.cssFloat = 'right'; - sorticon.style.marginLeft = '0.5em'; - headers[col].appendChild(sorticon); - headers[col].addEventListener( - 'click', - function (event) + this.column = i; + this.header = tbl.tHead.rows[0].cells[i]; + this.sorted = false; + this.direction = 1; + + var container = this.header.appendChild(document.createElement('div')); + container.setAttribute('class', 'sorticon'); + + this.icon = container.appendChild(document.createElement('img')); + this.icon.setAttribute('src', assetsurl + '/sortind.svg'); + this.icon.addEventListener('click', this.icon_click.bind(this)); + } + + TableSortHelper.prototype = (function () + { + function set_unsorted(obj) + { + obj.sorted = false; + obj.icon.setAttribute('src', assetsurl + '/sortind.svg'); + } + + function sort_table(obj) + { + var c = obj.header.cellIndex; + var numeric = obj.header.getAttribute('class') == 'numeric'; + var tbody = tbl.tBodies[0]; + var trows; + if (numeric) + { + trows = Array.prototype.slice.call(tbody.rows, 0).sort( + function (x, y) + { + return obj.direction * (parseFloat(x.cells[c].textContent) - parseFloat(y.cells[c].textContent)); + }); + } + else { - var imgsrc = sorticon.getAttribute('src'); - imgsrc = imgsrc.substr(imgsrc.lastIndexOf('/') + 1); - if (imgsrc != 'sortind.png') - dir = -dir; - if (dir < 0) - sorticon.setAttribute('src', assetsurl + '/sortdesc.png'); + trows = Array.prototype.slice.call(tbody.rows, 0).sort( + function (x, y) + { + return obj.direction * x.cells[c].textContent.localeCompare(y.cells[c].textContent); + }); + } + trows.forEach(function (row) { tbody.appendChild(row); }); + } + + return { + icon_click : function (event) + { + if (this.sorted) + this.direction = -this.direction; + if (this.direction < 0) + this.icon.setAttribute('src', assetsurl + '/sortdesc.svg'); else - sorticon.setAttribute('src', assetsurl + '/sortasc.png'); - for (var i = 0; i < headers.length; i++) + this.icon.setAttribute('src', assetsurl + '/sortasc.svg'); + this.sorted = true; + for (var i = 0; i < sorticons.length; i++) { - if (i != col) - headers[i].lastChild.setAttribute('src', assetsurl + '/sortind.png'); + if (i != this.column) + set_unsorted(sorticons[i]); } - sort_table(tbl, col, dir, headers[col].getAttribute('class') == 'numeric'); - }); - }(i)); - } + sort_table(this); + } + }; + })(); + + for (var i = 0; i < sorticons.length; i++) + sorticons[i] = new TableSortHelper(i); } diff --git a/scripts/minimaws/lib/assets/digest.js b/scripts/minimaws/lib/assets/digest.js new file mode 100644 index 00000000000..91285ff529f --- /dev/null +++ b/scripts/minimaws/lib/assets/digest.js @@ -0,0 +1,235 @@ +// license:BSD-3-Clause +// copyright-holders:Vas Crabb +'use strict'; + +function Crc32Digester() +{ + this.crc = 0xffffffff; +} + +Crc32Digester.prototype = (function () + { + var table = new Uint32Array(256); + (function () + { + for (var i = 0; i < 256; i++) + { + var crc = i; + for (var b = 0; b < 8; b++) + crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0x00000000); + table[i] = crc; + } + })(); + + return { + digest: function (data, view) + { + for (var i = 0; i < view.length; i++) + this.crc = (this.crc >>> 8) ^ table[(this.crc & 0x000000ff) ^ view[i]]; + }, + + finalise: function () + { + return this.crc ^ 0xffffffff; + } + }; + })(); + + +function Sha1Digester() +{ + this.st = new Uint32Array(5); + this.st[0] = 0xc3d2e1f0; + this.st[1] = 0x10325476; + this.st[2] = 0x98badcfe; + this.st[3] = 0xefcdab89; + this.st[4] = 0x67452301; + + this.cnt = new Uint32Array(2); + this.cnt[0] = 0; + this.cnt[1] = 0; + + this.buf = new DataView(new ArrayBuffer(64)); +} + +Sha1Digester.prototype = (function () + { + function rol(x, n) + { + return ((x << n) | (x >>> (32 - n))) & 0xffffffff; + } + + function b(data, i) + { + var r = data.getUint32(((i + 13) & 15) << 2, false); + r ^= data.getUint32(((i + 8) & 15) << 2, false); + r ^= data.getUint32(((i + 2) & 15) << 2, false); + r ^= data.getUint32((i & 15) << 2, false); + r = rol(r, 1); + data.setUint32((i & 15) << 2, r, false); + return r; + } + + function r0(data, d, i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + ((d[(i + 3) % 5] & (d[(i + 2) % 5] ^ d[(i + 1) % 5])) ^ d[(i + 1) % 5]) + data.getUint32(i << 2, false) + 0x5a827999 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + + function r1(data, d, i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + ((d[(i + 3) % 5] & (d[(i + 2) % 5] ^ d[(i + 1) % 5])) ^ d[(i + 1) % 5])+ b(data, i) + 0x5a827999 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + + function r2(data, d, i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + (d[(i + 3) % 5] ^ d[(i + 2) % 5] ^ d[(i + 1) % 5]) + b(data, i) + 0x6ed9eba1 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + + function r3(data, d, i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + (((d[(i + 3) % 5] | d[(i + 2) % 5]) & d[(i + 1) % 5]) | (d[(i + 3) % 5] & d[(i + 2) % 5])) + b(data, i) + 0x8f1bbcdc + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + + function r4(data, d, i) + { + d[i % 5] = 0xffffffff & (d[i % 5] + (d[(i + 3) % 5] ^ d[(i + 2) % 5] ^ d[(i + 1) % 5]) + b(data, i) + 0xca62c1d6 + rol(d[(i + 4) % 5], 5)); + d[(i + 3) % 5] = rol(d[(i + 3) % 5], 30); + } + + function process(st, data) + { + var d = new Uint32Array(st); + var i = 0; + while (i < 16) + r0(data, d, i++); + while (i < 20) + r1(data, d, i++); + while (i < 40) + r2(data, d, i++); + while (i < 60) + r3(data, d, i++); + while (i < 80) + r4(data, d, i++); + for (i = 0; i < st.length; i++) + st[i] = (st[i] + d[i]) & 0xffffffff; + } + + return { + digest: function (data, view) + { + var residual = this.cnt[0]; + this.cnt[0] = (this.cnt[0] + (view.length << 3)) & 0xffffffff; + if (residual > this.cnt[0]) + this.cnt[1]++; + this.cnt[1] += (view.length >>> 29); + residual = (residual >>> 3) & 63; + var offset = 0; + if ((residual + view.length) >= 64) + { + if (residual > 0) + { + for (offset = 0; (offset + residual) < 64; offset++) + this.buf.setUint8(offset + residual, view[offset]); + process(this.st, this.buf); + residual = 0; + } + for ( ; (view.length - offset) >= 64; offset += 64) + process(this.st, new DataView(data, offset, 64)); + } + for ( ; offset < view.length; residual++, offset++) + this.buf.setUint8(residual, view[offset]); + }, + + finalise : function () + { + var lenbuf = new ArrayBuffer(8); + var lenview = new DataView(lenbuf); + lenview.setUint32(0, this.cnt[1], false); + lenview.setUint32(4, this.cnt[0], false); + var padbuf = new ArrayBuffer(64 - (63 & ((this.cnt[0] >>> 3) + 8))); + var padview = new Uint8Array(padbuf); + padview[0] = 0x80; + for (var i = 1; i < padview.length; i++) + padview[i] = 0x00; + this.digest(padbuf, padview); + this.digest(lenbuf, new Uint8Array(lenbuf)); + var result = new Array(20); + for (var i = 0; i < 20; i++) + result[i] = (this.st[4 - (i >>> 2)] >> ((3 - (i & 3)) << 3)) & 0x000000ff; + return result.map(x => x.toString(16).padStart(2, '0')).join(''); + } + }; + })(); + + +function StuckBitsDigester(max = 4) +{ + this.offset = 0; + this.state = [ ]; + for (var i = 0; i <= max; i++) + { + var bytes = 1 << i; + this.state.push({ filled: 0, bits: new Uint8Array(bytes), mask: new Uint8Array(bytes) }); + } +} + +StuckBitsDigester.prototype = (function () + { + return { + digest: function (data, view) + { + for (var i = 0; i < view.length; i++) + { + var data = view[i]; + for (var j = 0; j < this.state.length; j++) + { + var detail = this.state[j]; + var bytes = 1 << j; + var o = this.offset % bytes; + var f = o == (bytes - 1); + if (!detail.filled) + { + detail.bits[o] = data; + detail.mask[o] = 0xff; + if (f) + detail.filled = 1; + } + else + { + detail.mask[o] &= ~(data ^ detail.bits[o]); + detail.bits[o] &= detail.mask[o]; + } + if (o == (bytes - 1)) + detail.filled += 1; + } + this.offset = (this.offset + 1) % (1 << (this.state.length - 1)); + } + }, + + finalise: function () + { + var result = [ ]; + var lastresult = null; + for (var i = 0; (i < this.state.length) && (this.state[i].filled >= 4); i++) + { + var detail = this.state[i]; + var bytes = 1 << i; + for (var j = 0; j < bytes; j++) + { + var mask = detail.mask[j]; + if (mask && ((lastresult === null) || (mask != lastresult.mask[j % (1 << (lastresult.mask.length - 1))]))) + { + lastresult = { bits: detail.bits, mask: detail.mask }; + result.push(lastresult); + break; + } + } + } + return result; + } + }; + })(); diff --git a/scripts/minimaws/lib/assets/disclosedown.svg b/scripts/minimaws/lib/assets/disclosedown.svg new file mode 100644 index 00000000000..bb66c7f5de9 --- /dev/null +++ b/scripts/minimaws/lib/assets/disclosedown.svg @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="16" width="16"> + <path d="m 0,5 h 16 l -8,10 z" /> +</svg> diff --git a/scripts/minimaws/lib/assets/discloseup.svg b/scripts/minimaws/lib/assets/discloseup.svg new file mode 100644 index 00000000000..010065a446b --- /dev/null +++ b/scripts/minimaws/lib/assets/discloseup.svg @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="16" width="16"> + <path d="m 5,0 v 16 l 10,-8 z" /> +</svg> diff --git a/scripts/minimaws/lib/assets/machine.js b/scripts/minimaws/lib/assets/machine.js index d0774d02c0a..b488ae9e9a4 100644 --- a/scripts/minimaws/lib/assets/machine.js +++ b/scripts/minimaws/lib/assets/machine.js @@ -1,9 +1,11 @@ // license:BSD-3-Clause // copyright-holders:Vas Crabb +'use strict'; var slot_info = Object.create(null); var bios_sets = Object.create(null); var machine_flags = Object.create(null); +var softwarelist_info = Object.create(null); function make_slot_popup_id(name) { return ('select-slot-choice-' + name).replace(/:/g, '-'); } @@ -136,7 +138,7 @@ var fetch_bios_sets = (function () { pending[device] = true; var req = new XMLHttpRequest(); - req.open('GET', appurl + 'rpc/bios/' + device, true); + req.open('GET', appurl + 'rpc/bios/' + encodeURIComponent(device), true); req.responseType = 'json'; req.onload = function () @@ -171,7 +173,7 @@ var fetch_machine_flags = (function () { pending[device] = true; var req = new XMLHttpRequest(); - req.open('GET', appurl + 'rpc/flags/' + device, true); + req.open('GET', appurl + 'rpc/flags/' + encodeURIComponent(device), true); req.responseType = 'json'; req.onload = function () @@ -197,6 +199,47 @@ var fetch_machine_flags = (function () })(); +var fetch_softwarelist_info = (function () + { + var pending = Object.create(null); + return function (device) + { + if (!Object.prototype.hasOwnProperty.call(softwarelist_info, device) && !Object.prototype.hasOwnProperty.call(pending, device)) + { + pending[device] = true; + var req = new XMLHttpRequest(); + req.open('GET', appurl + 'rpc/softwarelists/' + encodeURIComponent(device), true); + req.responseType = 'json'; + req.onload = + function () + { + delete pending[device]; + if (req.status == 200) + { + softwarelist_info[device] = req.response; + var slotslist = document.getElementById('list-slot-options'); + if (slotslist) + { + for (var item = slotslist.firstChild; item; item = item.nextSibling) + { + if ((item.nodeName == 'DT') && (item.getAttribute('data-slotcard') == device)) + { + var slotname = item.getAttribute('data-slotname'); + add_softwarelist_rows( + device, + slotname, + document.getElementById('select-slot-choice-' + slotname.replace(/:/g, '-')).value); + } + } + } + } + }; + req.send(); + } + }; + })(); + + function add_flag_rows(table, device) { var sorted_features = Object.keys(machine_flags[device].features).sort(); @@ -228,6 +271,51 @@ function add_flag_rows(table, device) } +function add_softwarelist_rows(device, slot, card) +{ + var sorted_softwarelists = Object.keys(softwarelist_info[device]).sort(); + var cardtag = slot + ':' + card; + if (sorted_softwarelists.length) + { + var table = document.getElementById('tbl-softwarelists').tBodies[0]; + sorted_softwarelists.forEach( + function (tag) + { + var cell, link; + var info = softwarelist_info[device][tag]; + var href = appurl + 'softwarelist/' + encodeURIComponent(info.shortname); + var row = table.appendChild(document.createElement('tr')); + row.setAttribute('id', 'row-softwarelists-' + (cardtag + tag).replace(/:/g, '-')) + row.appendChild(document.createElement('td')).textContent = cardtag; + link = row.appendChild(document.createElement('td')).appendChild(document.createElement('a')); + link.setAttribute('href', href); + link.textContent = info.shortname; + link = row.appendChild(document.createElement('td')).appendChild(document.createElement('a')); + link.setAttribute('href', href); + link.textContent = info.description; + row.appendChild(document.createElement('td')).textContent = info.status; + total = info.total; + cell = row.appendChild(document.createElement('td')); + cell.textContent = total.toString(10); + cell.style.textAlign = 'right'; + if (!total) + total = 1; + cell = row.appendChild(document.createElement('td')); + cell.textContent = (info.supported * 100.0 / total).toFixed(1) + '%'; + cell.style.textAlign = 'right'; + cell = row.appendChild(document.createElement('td')); + cell.textContent = (info.partiallysupported * 100.0 / total).toFixed(1) + '%'; + cell.style.textAlign = 'right'; + cell = row.appendChild(document.createElement('td')); + cell.textContent = (info.unsupported * 100.0 / total).toFixed(1) + '%'; + cell.style.textAlign = 'right'; + }); + document.getElementById('heading-softwarelists').style.removeProperty('display'); + document.getElementById('tbl-softwarelists').style.removeProperty('display'); + } +} + + function add_bios_row(slot, table, device) { var sorted_sets = Object.keys(bios_sets[device]).sort(); @@ -394,6 +482,16 @@ function make_slot_change_handler(name, slot, defaults, dfltbtn) slotslist.removeChild(next); } + // clear out any software lists from previous selection + var softwarelist_rowid_prefix = 'row-softwarelists-' + prefix.replace(/:/g, '-'); + for (var candidate = document.getElementById('tbl-softwarelists').tBodies[0].rows[0]; candidate; ) + { + var next = candidate.nextSibling; + if ((candidate.nodeName == 'TR') && candidate.getAttribute('id').startsWith(softwarelist_rowid_prefix)) + candidate.parentNode.removeChild(candidate); + candidate = next; + } + if (selection === null) { // no selection, remove the slot card details table @@ -413,7 +511,7 @@ function make_slot_change_handler(name, slot, defaults, dfltbtn) row.appendChild(document.createElement('th')).textContent = 'Short name:'; var link = row.appendChild(document.createElement('td')).appendChild(document.createElement('a')); link.textContent = selection.device; - link.setAttribute('href', appurl + 'machine/' + selection.device); + link.setAttribute('href', appurl + 'machine/' + encodeURIComponent(selection.device)); // if we have emulation flags, populate now, otherwise fetch asynchronously if (!Object.prototype.hasOwnProperty.call(machine_flags, selection.device)) @@ -427,6 +525,12 @@ function make_slot_change_handler(name, slot, defaults, dfltbtn) else add_bios_row(slotname, tbl, selection.device); + // if we have software list info, populate now, otherweise fetch asynchronously + if (!Object.prototype.hasOwnProperty.call(softwarelist_info, selection.device)) + fetch_softwarelist_info(selection.device); + else + add_softwarelist_rows(selection.device, slotname, choice); + // drop the details table into the list if (def.firstChild) def.replaceChild(tbl, def.firstChild); @@ -487,7 +591,7 @@ function fetch_slots(machine) function make_request(device) { var req = new XMLHttpRequest(); - req.open('GET', appurl + 'rpc/slots/' + device, true); + req.open('GET', appurl + 'rpc/slots/' + encodeURIComponent(device), true); req.responseType = 'json'; req.onload = function () diff --git a/scripts/minimaws/lib/assets/romident.js b/scripts/minimaws/lib/assets/romident.js new file mode 100644 index 00000000000..892894ced56 --- /dev/null +++ b/scripts/minimaws/lib/assets/romident.js @@ -0,0 +1,642 @@ +// license:BSD-3-Clause +// copyright-holders:Vas Crabb +'use strict'; + +var matched_names = new Array(); +var unmatched_names = new Array(); +var dump_info = Object.create(null); +var machine_info = Object.create(null); +var softwarelist_info = Object.create(null); +var software_info = Object.create(null); + + +function get_chd_sha1(header) +{ + if (header.byteLength < 16) + return null; + if (String.fromCharCode.apply(null, new Uint8Array(header, 0, 8)) != 'MComprHD') + return null; + var view = new DataView(header); + var headerlen = view.getUint32(8, false); + var version = view.getUint32(12, false); + var sha1offs; + if (version == 3) + { + if (headerlen != 120) + return null; + sha1offs = 80; + } + else if (version == 4) + { + if (headerlen != 108) + return null; + sha1offs = 48; + } + else if (version == 5) + { + if (headerlen != 124) + return null; + sha1offs = 84; + } + else + { + return null; + } + if (header.byteLength < (sha1offs + 20)) + return null; + return Array.from(new Uint8Array(header, sha1offs, 20)).map(x => x.toString(16).padStart(2, '0')).join(''); +} + + +function get_sha1_group(sha1) +{ + if (Object.hasOwnProperty.call(dump_info, sha1)) + { + return dump_info[sha1]; + } + else + { + var result = new Object(); + result.crc = Object.create(null); + dump_info[sha1] = result; + return result; + } +} + + +function add_matches(table, names, matches) +{ + for (var i = 0; i < names.length; i++) + { + var row = table.appendChild(document.createElement('tr')); + var name = row.appendChild(document.createElement('th')); + name.textContent = names[i]; + if (matches !== null) + { + name.setAttribute('rowspan', matches.length); + row.appendChild(document.createElement('td')).textContent = matches[0].name; + var bad = row.appendChild(document.createElement('td')); + if (matches[0].bad) + bad.textContent = 'BAD'; + for (var j = 1; j < matches.length; j++) + { + row = table.appendChild(document.createElement('tr')); + row.appendChild(document.createElement('td')).textContent = matches[j].name; + bad = row.appendChild(document.createElement('td')); + if (matches[j].bad) + bad.textContent = 'BAD'; + } + } + } +} + + +function add_software_matches(table, names, part, matches) +{ + for (var i = 0; i < names.length; i++) + { + var row = table.appendChild(document.createElement('tr')); + var name = row.appendChild(document.createElement('th')); + name.textContent = names[i]; + if (matches !== null) + { + name.setAttribute('rowspan', matches.length); + row.appendChild(document.createElement('td')).textContent = part; + row.appendChild(document.createElement('td')).textContent = matches[0].name; + var bad = row.appendChild(document.createElement('td')); + if (matches[0].bad) + bad.textContent = 'BAD'; + for (var j = 1; j < matches.length; j++) + { + row = table.appendChild(document.createElement('tr')); + row.appendChild(document.createElement('td')).textContent = matches[j].name; + bad = row.appendChild(document.createElement('td')); + if (matches[j].bad) + bad.textContent = 'BAD'; + } + } + } +} + + +function add_unmatched(names, crc, sha1) +{ + var table; + if (unmatched_names.length > 0) + { + table = document.getElementById('table-unmatched'); + } + else + { + var div = document.getElementById('div-unmatched'); + var heading = div.appendChild(document.createElement('h2')); + heading.textContent = 'Unmatched'; + table = div.appendChild(document.createElement('table')); + table.setAttribute('id', 'table-unmatched'); + make_collapsible(heading, table); + } + var content; + if (crc === null) + { + content = 'SHA1(' + sha1 + ')'; + } + else + { + if (crc < 0) + crc = 0xffffffff + crc + 1; + content = 'CRC(' + crc.toString(16).padStart(8, '0') + ') SHA1(' + sha1 + ')'; + } + for (var i = 0; i < names.length; i++) + { + var row = table.appendChild(document.createElement('tr')); + row.appendChild(document.createElement('th')).textContent = names[i]; + row.appendChild(document.createElement('td')).textContent = content; + unmatched_names.push(names[i]); + } +} + + +function add_issues(name) +{ + var div = document.getElementById('div-issues'); + var list; + if (!div.hasChildNodes()) + { + var heading = div.appendChild(document.createElement('h2')); + heading.textContent = 'Potential Issues'; + list = div.appendChild(document.createElement('dl')); + make_collapsible(heading, list); + } + else + { + list = div.lastChild; + } + list.appendChild(document.createElement('dt')).textContent = name; + var table = list.appendChild(document.createElement('dd')).appendChild(document.createElement('table')); + table.setAttribute('class', 'sysinfo'); + return table; +} + + +function add_stuck_bits(table, stuck) +{ + function format_stuck_bits(bits, mask) + { + var result = ''; + for (var i = 0; i < bits.length; i++) + { + if (i > 0) + result += ' '; + for (var j = 0; j < 8; j++) + { + if (!((mask[i] >> (7 - j)) & 0x01)) + result += '-'; + else if ((bits[i] >> (7 - j)) & 0x01) + result += '1'; + else + result += '0'; + } + } + var cell = document.createElement('td'); + cell.appendChild(document.createElement('tt')).textContent = result; + return cell; + } + + var row = table.appendChild(document.createElement('tr')); + var header = row.appendChild(document.createElement('th')); + header.textContent = 'Fixed data bits:'; + header.setAttribute('rowspan', stuck.length); + row.appendChild(format_stuck_bits(stuck[0].bits, stuck[0].mask)); + for (var i = 1; i < stuck.length; i++) + { + row = table.appendChild(document.createElement('tr')); + row.appendChild(format_stuck_bits(stuck[i].bits, stuck[i].mask)); + } +} + + +function get_machine_table(shortname, description) +{ + if (Object.hasOwnProperty.call(machine_info, shortname)) + { + return machine_info[shortname]; + } + else + { + var div = document.getElementById('div-machines'); + if (!div.hasChildNodes()) + div.appendChild(document.createElement('h2')).textContent = 'Machines'; + var heading = div.appendChild(document.createElement('h3')); + var link = heading.appendChild(document.createElement('a')); + link.textContent = description; + link.setAttribute('href', appurl + 'machine/' + encodeURIComponent(shortname)); + var table = div.appendChild(document.createElement('table')); + machine_info[shortname] = table; + add_matches(table, matched_names, null); + make_collapsible(heading, table); + return table; + } +} + + +function get_softwarelist_div(shortname, description) +{ + if (Object.hasOwnProperty.call(softwarelist_info, shortname)) + { + return softwarelist_info[shortname]; + } + else + { + software_info[shortname] = Object.create(null) + var div = document.getElementById('div-software'); + if (!div.hasChildNodes()) + div.appendChild(document.createElement('h2')).textContent = 'Software'; + div = div.appendChild(document.createElement('div')); + var heading = div.appendChild(document.createElement('h3')); + var link = heading.appendChild(document.createElement('a')); + link.textContent = description; + link.setAttribute('href', appurl + 'softwarelist/' + encodeURIComponent(shortname)); + softwarelist_info[shortname] = div; + return div; + } +} + + +function get_software_table(softwarelist, shortname, description) +{ + if (Object.hasOwnProperty.call(software_info, softwarelist) && Object.hasOwnProperty.call(software_info[softwarelist], shortname)) + { + return software_info[softwarelist][shortname]; + } + else + { + var div = softwarelist_info[softwarelist]; + var heading = div.appendChild(document.createElement('h4')); + var link = heading.appendChild(document.createElement('a')); + link.textContent = description; + link.setAttribute('href', appurl + 'softwarelist/' + encodeURIComponent(softwarelist) + '/' + encodeURIComponent(shortname)); + var table = div.appendChild(document.createElement('table')); + software_info[softwarelist][shortname] = table; + add_software_matches(table, matched_names, null, null); + return table; + } +} + + +function request_dumps(name, group, crc, sha1, url, progress) +{ + var req = new XMLHttpRequest(); + req.responseType = 'json'; + req.onload = + function () + { + if (req.status == 200) + { + var machines = Object.create(null); + var software = Object.create(null); + var matched = Object.keys(req.response.machines); + var softwarelists = Object.keys(req.response.software); + + if (matched.length > 0) + { + Object.keys(machine_info).forEach( + function (shortname) + { + var machine_table = machine_info[shortname]; + if (Object.hasOwnProperty.call(req.response.machines, shortname)) + { + machines[shortname] = req.response.machines[shortname].matches; + add_matches(machine_table, group.names, req.response.machines[shortname].matches); + } + else + { + add_matches(machine_table, group.names, null); + } + }); + if (softwarelists.length <= 0) + { + Object.keys(software_info).forEach( + function (listname) + { + Object.keys(software_info[listname]).forEach( + function (softwarename) + { + var software_table = software_info[listname][softwarename]; + add_software_matches(software_table, group.names, null, null); + }); + }); + } + matched.forEach( + function (shortname) + { + if (!Object.hasOwnProperty.call(machine_info, shortname)) + { + var machine_details = req.response.machines[shortname]; + var machine_table = get_machine_table(shortname, machine_details.description); + machines[shortname] = req.response.machines[shortname].matches; + add_matches(machine_table, group.names, machine_details.matches); + } + }); + } + + if (softwarelists.length > 0) + { + if (matched.length <= 0) + { + Object.keys(machine_info).forEach( + function (shortname) + { + var machine_table = machine_info[shortname]; + add_matches(machine_table, group.names, null); + }); + } + Object.keys(software_info).forEach( + function (listname) + { + var haslist = Object.hasOwnProperty.call(req.response.software, listname); + Object.keys(software_info[listname]).forEach( + function (softwarename) + { + var software_table = software_info[listname][softwarename]; + if (haslist && Object.hasOwnProperty.call(req.response.software[listname].software, softwarename)) + { + if (!Object.hasOwnProperty.call(software, listname)) + software[listname] = Object.create(null); + if (!Object.hasOwnProperty.call(software[listname], softwarename)) + software[listname][softwarename] = Object.create(null); + var software_details = req.response.software[listname].software[softwarename]; + Object.keys(software_details.parts).forEach( + function (partname) + { + var matches = software_details.parts[partname].matches; + software[listname][softwarename][partname] = matches; + add_software_matches(software_table, group.names, partname, matches); + }); + } + else + { + add_software_matches(software_table, group.names, null, null); + } + }); + }); + softwarelists.forEach( + function (listname) + { + var haslist = Object.hasOwnProperty.call(software_info, listname); + var softwarelist_details = req.response.software[listname]; + var softwarelist_div = get_softwarelist_div(listname, softwarelist_details.description); + Object.keys(softwarelist_details.software).forEach( + function (softwarename) + { + if (!haslist || !Object.hasOwnProperty.call(software_info[listname], softwarename)) + { + if (!Object.hasOwnProperty.call(software, listname)) + software[listname] = Object.create(null); + if (!Object.hasOwnProperty.call(software[listname], softwarename)) + software[listname][softwarename] = Object.create(null); + var software_details = softwarelist_details.software[softwarename]; + var software_table = get_software_table(listname, softwarename, software_details.description); + Object.keys(software_details.parts).forEach( + function (partname) + { + var matches = software_details.parts[partname].matches; + software[listname][softwarename][partname] = matches; + add_software_matches(software_table, group.names, partname, matches); + }); + } + }); + }); + } + + if ((matched.length > 0) | (softwarelists.length > 0)) + { + for (var i = 0; i < group.names.length; i++) + matched_names.push(group.names[i]); + } + else + { + add_unmatched(group.names, crc, sha1); + } + + group.machines = machines; + group.software = software; + progress.parentNode.removeChild(progress); + } + else + { + progress.textContent = 'Error identifying \u201C' + name + '\u201D: HTTP status code ' + req.status.toString(10); + if (crc !== null) + delete dump_info[sha1].crc[crc]; + else + delete dump_info[sha1].disk; + } + }; + req.onerror = + function () + { + progress.textContent = 'Error identifying \u201C' + name + '\u201D'; + if (crc !== null) + delete dump_info[sha1].crc[crc]; + else + delete dump_info[sha1].disk; + }; + req.onabort = + function () + { + progress.textContent = 'Error identifying \u201C' + name + '\u201D: server request aborted'; + if (crc !== null) + delete dump_info[sha1].crc[crc]; + else + delete dump_info[sha1].disk; + }; + req.open('GET', url); + req.send(); +} + + +function add_name(group, name, crc, sha1) +{ + if (group.names.indexOf(name) < 0) + { + group.names.push(name); + + if (group.hasOwnProperty('issues')) + { + var issues = group.issues; + if (issues !== null) + issues.parentNode.parentNode.insertBefore(document.createElement('dt'), issues.parentNode).textContent = name; + } + + if (group.hasOwnProperty('machines')) + { + var machines = group.machines; + var software = group.software; + var names = [ name ]; + if ((Object.keys(machines).length > 0) || (Object.keys(software).length > 0)) + { + Object.keys(machine_info).forEach( + function (shortname) + { + var machine_table = machine_info[shortname]; + if (Object.hasOwnProperty.call(machines, shortname)) + add_matches(machine_table, names, machines[shortname]); + else + add_matches(machine_table, names, null); + }); + + Object.keys(software_info).forEach( + function (listname) + { + var haslist = Object.hasOwnProperty.call(software, listname); + Object.keys(software_info[listname]).forEach( + function (softwarename) + { + var software_table = software_info[listname][softwarename]; + if (haslist && Object.hasOwnProperty.call(software[listname], softwarename)) + { + Object.keys(software[listname][softwarename]).forEach( + function (partname) + { + add_software_matches(software_table, names, partname, software[listname][softwarename][partname]); + }); + } + else + { + add_software_matches(software_table, names, null, null); + } + }); + }); + + matched_names.push(name); + } + else + { + add_unmatched(names, crc, sha1); + } + } + } +} + + +function identify_file(file, trychd, progress) +{ + var digested = 0; + var crcproc = new Crc32Digester(); + var sha1proc = new Sha1Digester(); + var stuckbitsproc = new StuckBitsDigester(); + + function process_chunk(e) + { + if (e.target.error === null) + { + var data = e.target.result; + if ((digested == 0) && trychd) + { + var chdsha1 = get_chd_sha1(data); + if (chdsha1 !== null) + { + var sha1grp = get_sha1_group(chdsha1); + if (!sha1grp.hasOwnProperty('disk')) + { + var diskgrp = new Object(); + diskgrp.names = [ file.name ]; + sha1grp.disk = diskgrp; + request_dumps(file.name, diskgrp, null, chdsha1, appurl + 'rpc/diskdumps?sha1=' + chdsha1, progress); + } + else + { + add_name(sha1grp.disk, file.name, crc, sha1); + progress.parentNode.removeChild(progress); + } + return; + } + } + var view = new Uint8Array(data); + crcproc.digest(data, view); + sha1proc.digest(data, view); + stuckbitsproc.digest(data, view); + digested += data.byteLength; + if (digested < file.size) + { + read_block(); + } + else + { + var crc = crcproc.finalise(); + var sha1 = sha1proc.finalise(); + var stuckbits = stuckbitsproc.finalise(); + + var sha1grp = get_sha1_group(sha1); + var crcgrp; + if (!Object.hasOwnProperty.call(sha1grp.crc, crc)) + { + crcgrp = new Object(); + crcgrp.names = [ file.name ]; + sha1grp.crc[crc] = crcgrp; + if (stuckbits.length) + { + crcgrp.issues = add_issues(file.name); + add_stuck_bits(crcgrp.issues, stuckbits); + } + else + { + crcgrp.issues = null; + } + var crcstr = ((crc < 0) ? (0xffffffff + crc + 1) : crc).toString(16).padStart(8, '0'); + request_dumps(file.name, crcgrp, crc, sha1, appurl + 'rpc/romdumps?crc=' + crcstr + '&sha1=' + sha1, progress); + } + else + { + crcgrp = sha1grp.crc[crc]; + add_name(crcgrp, file.name, crc, sha1); + progress.parentNode.removeChild(progress); + } + } + } + else + { + progress.textContent = 'Error identifying \u201C' + file.name + '\u201D: ' + e.target.error; + } + } + + function read_block() + { + var remaining = file.size - digested; + var chunk = file.slice(digested, digested + Math.min(remaining, 65536)); + var reader = new FileReader(); + reader.onload = process_chunk; + reader.readAsArrayBuffer(chunk); + }; + + progress.textContent = 'Identifying \u201C' + file.name + '\u201D\u2026'; + read_block(); +} + + +function add_dump_files(files) +{ + var prog = document.getElementById('div-progress'); + for (var i = 0; i < files.length; i++) + { + var name = files[i].name; + var dot = name.lastIndexOf('.'); + var trychd = (dot > 0) && (name.substring(dot + 1).toLowerCase() == 'chd'); + identify_file(files[i], trychd, prog.appendChild(document.createElement('p'))); + } +} + + +function div_dropzone_dragover(e) +{ + e.stopPropagation(); + e.preventDefault(); + e.dataTransfer.dropEffect = 'link'; +} + + +function div_dropzone_drop(e) +{ + e.stopPropagation(); + e.preventDefault(); + add_dump_files(e.dataTransfer.files); +} diff --git a/scripts/minimaws/lib/assets/sortasc.png b/scripts/minimaws/lib/assets/sortasc.png Binary files differdeleted file mode 100644 index 29b893cdd02..00000000000 --- a/scripts/minimaws/lib/assets/sortasc.png +++ /dev/null diff --git a/scripts/minimaws/lib/assets/sortasc.svg b/scripts/minimaws/lib/assets/sortasc.svg new file mode 100644 index 00000000000..c2ff41afccb --- /dev/null +++ b/scripts/minimaws/lib/assets/sortasc.svg @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="16" width="30"> + <path d="m 14,16 h 16 l -8,-16 z" /> +</svg> diff --git a/scripts/minimaws/lib/assets/sortdesc.png b/scripts/minimaws/lib/assets/sortdesc.png Binary files differdeleted file mode 100644 index 63f48fc7213..00000000000 --- a/scripts/minimaws/lib/assets/sortdesc.png +++ /dev/null diff --git a/scripts/minimaws/lib/assets/sortdesc.svg b/scripts/minimaws/lib/assets/sortdesc.svg new file mode 100644 index 00000000000..4a10b3f6e47 --- /dev/null +++ b/scripts/minimaws/lib/assets/sortdesc.svg @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="16" width="30"> + <path d="m 14,0 h 16 l -8,16 z" /> +</svg> diff --git a/scripts/minimaws/lib/assets/sortind.png b/scripts/minimaws/lib/assets/sortind.png Binary files differdeleted file mode 100644 index 773dd6aec02..00000000000 --- a/scripts/minimaws/lib/assets/sortind.png +++ /dev/null diff --git a/scripts/minimaws/lib/assets/sortind.svg b/scripts/minimaws/lib/assets/sortind.svg new file mode 100644 index 00000000000..6e351b6a6fb --- /dev/null +++ b/scripts/minimaws/lib/assets/sortind.svg @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="16" width="30"> + <path d="m 0,16 h 16 l -8,-16 z" /> + <path d="m 14,0 h 16 l -8,16 z" /> +</svg> diff --git a/scripts/minimaws/lib/assets/style.css b/scripts/minimaws/lib/assets/style.css index 75ea0fa93de..9ba6631a783 100644 --- a/scripts/minimaws/lib/assets/style.css +++ b/scripts/minimaws/lib/assets/style.css @@ -5,7 +5,14 @@ th { text-align: left; background-color: #ddd; padding: 0.25em } td { padding-left: 0.25em; padding-right: 0.25em } +img[class=disclosure] { height: 0.7em; margin-right: 0.25em; vertical-align: baseline } + +div[class=sorticon] { float: right; height: 100%; margin-left: 0.5em } +div[class=sorticon] img { height: 0.7em; vertical-align: baseline } + table[class=sysinfo] th { text-align: right } +div[class=dropzone] { padding: 0 1em; border-width: 2px; border-radius: 1em; border-style: dashed } + dl[id=list-slot-options] dt { font-weight: bold; margin-top: 1em } dl[id=list-slot-options] dd table { margin-top: 0.5em; margin-bottom: 1em } diff --git a/scripts/minimaws/lib/auxverbs.py b/scripts/minimaws/lib/auxverbs.py index c68c528af6f..f4a4ebbe70c 100644 --- a/scripts/minimaws/lib/auxverbs.py +++ b/scripts/minimaws/lib/auxverbs.py @@ -1,11 +1,207 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb from . import dbaccess +import codecs +import hashlib +import os +import os.path +import struct import sys +import zipfile +import zlib + + +class _Identifier(object): + def __init__(self, dbcurs, **kwargs): + super().__init__(**kwargs) + self.dbcurs = dbcurs + self.shortnamewidth = 0 + self.pathwidth = 0 + self.labelwidth = 0 + self.machines = { } + self.software = { } + self.unmatched = [ ] + + def processPath(self, path, depth=0): + try: + if not os.path.isdir(path): + self.processFile(path) + elif depth > 5: + sys.stderr.write('Not examining \'%s\' - maximum depth exceeded\n') + else: + for name in os.listdir(path): + self.processPath(os.path.join(path, name), depth + 1) + except BaseException as e: + sys.stderr.write('Error identifying \'%s\': %s\n' % (path, e)) + + def printResults(self): + nw = self.shortnamewidth - (self.shortnamewidth % 4) + 4 + pw = self.pathwidth - (self.pathwidth % 4) + 4 + lw = self.labelwidth - (self.labelwidth % 4) + 4 + first = True + for shortname, romset in sorted(self.machines.items()): + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('%-*s%s\n' % (nw, shortname, romset[0])) + self.printMatches(romset[1], pw, lw) + for softwarelist, listinfo in sorted(self.software.items()): + for shortname, softwareinfo in sorted(listinfo[1].items()): + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('%-*s%s\n' % (nw, '%s:%s' % (softwarelist, shortname), softwareinfo[0])) + for part, partinfo in sorted(softwareinfo[1].items()): + if partinfo[0] is not None: + sys.stdout.write('%-*s%s\n' % (nw, ' ' + part, partinfo[0])) + else: + sys.stdout.write(' %s\n' % (part, )) + self.printMatches(partinfo[1], pw, lw) + if self.unmatched: + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('Unmatched\n') + for path, crc, sha1 in self.unmatched: + if crc is not None: + sys.stdout.write(' %-*sCRC(%08x) SHA1(%s)\n' % (pw, path, crc, sha1)) + else: + sys.stdout.write(' %-*sSHA1(%s)\n' % (pw, path, sha1)) + + def getMachineMatches(self, shortname, description): + result = self.machines.get(shortname) + if result is None: + result = (description, [ ]) + self.machines[shortname] = result + return result[1] + + def getSoftwareMatches(self, softwarelist, softwarelistdescription, shortname, description, part, part_id): + listinfo = self.software.get(softwarelist) + if listinfo is None: + listinfo = (softwarelistdescription, { }) + self.software[softwarelist] = listinfo + softwareinfo = listinfo[1].get(shortname) + if softwareinfo is None: + softwareinfo = (description, { }) + listinfo[1][shortname] = softwareinfo + partinfo = softwareinfo[1].get(part) + if partinfo is None: + partinfo = (part_id, [ ]) + softwareinfo[1][part] = partinfo + return partinfo[1] + + def processRomFile(self, path, f): + crc, sha1 = self.digestRom(f) + matched = False + for shortname, description, label, bad in self.dbcurs.get_rom_dumps(crc, sha1): + matched = True + self.shortnamewidth = max(len(shortname), self.shortnamewidth) + self.labelwidth = max(len(label), self.labelwidth) + self.getMachineMatches(shortname, description).append((path, label, bad)) + for softwarelist, softwarelistdescription, shortname, description, part, part_id, label, bad in self.dbcurs.get_software_rom_dumps(crc, sha1): + matched = True + self.shortnamewidth = max(len(softwarelist) + 1 + len(shortname), 2 + len(part), self.shortnamewidth) + self.labelwidth = max(len(label), self.labelwidth) + self.getSoftwareMatches(softwarelist, softwarelistdescription, shortname, description, part, part_id).append((path, label, bad)) + if not matched: + self.unmatched.append((path, crc, sha1)) + + def processChd(self, path, sha1): + matched = False + for shortname, description, label, bad in self.dbcurs.get_disk_dumps(sha1): + matched = True + self.shortnamewidth = max(len(shortname), self.shortnamewidth) + self.labelwidth = max(len(label), self.labelwidth) + self.getMachineMatches(shortname, description).append((path, label, bad)) + for softwarelist, softwarelistdescription, shortname, description, part, part_id, label, bad in self.dbcurs.get_software_disk_dumps(sha1): + matched = True + self.shortnamewidth = max(len(softwarelist) + 1 + len(shortname), 2 + len(part), self.shortnamewidth) + self.labelwidth = max(len(label), self.labelwidth) + self.getSoftwareMatches(softwarelist, softwarelistdescription, shortname, description, part, part_id).append((path, label, bad)) + if not matched: + self.unmatched.append((path, None, sha1)) + + def processFile(self, path): + if os.path.splitext(path)[1].lower() != '.chd': + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path, 'r') as zip: + for info in zip.infolist(): + if info.filename[-1] != '/': + with zip.open(info, mode='r') as f: + self.processRomFile(path + '/' + info.filename, f) + else: + with open(path, mode='rb', buffering=0) as f: + self.processRomFile(path, f) + else: + with open(path, mode='rb') as f: + sha1 = self.probeChd(f) + if sha1 is None: + f.seek(0) + self.processRomFile(path, f) + else: + self.processChd(path, sha1) + self.pathwidth = max(len(path), self.pathwidth) + + @staticmethod + def iterateBlocks(f, s=65536): + while True: + buf = f.read(s) + if buf: + yield buf + else: + break + + @staticmethod + def digestRom(f): + crc = zlib.crc32(bytes()) + sha = hashlib.sha1() + for block in _Identifier.iterateBlocks(f): + crc = zlib.crc32(block, crc) + sha.update(block) + return crc & 0xffffffff, sha.hexdigest() + + @staticmethod + def probeChd(f): + buf = f.read(16) + if (len(buf) != 16) or (buf[:8] != b'MComprHD'): + return None + headerlen, version = struct.unpack('>II', buf[8:]) + if version == 3: + if headerlen != 120: + return None + sha1offs = 80 + elif version == 4: + if headerlen != 108: + return None + sha1offs = 48 + elif version == 5: + if headerlen != 124: + return None + sha1offs = 84 + else: + return None + f.seek(sha1offs) + if f.tell() != sha1offs: + return None + buf = f.read(20) + if len(buf) != 20: + return None + return codecs.getencoder('hex_codec')(buf)[0].decode('ascii') + + @staticmethod + def printMatches(matches, pathwidth, labelwidth): + for path, label, bad in matches: + if bad: + sys.stdout.write(' %-*s= %-*s(BAD)\n' % (pathwidth, path, labelwidth, label)) + else: + sys.stdout.write(' %-*s= %s\n' % (pathwidth, path, label)) def do_listfull(options): @@ -68,6 +264,7 @@ def do_listbrothers(options): dbcurs.close() dbconn.close() + def do_listaffected(options): dbconn = dbaccess.QueryConnection(options.database) dbcurs = dbconn.cursor() @@ -81,3 +278,14 @@ def do_listaffected(options): sys.stderr.write('No matching systems found for \'%s\'\n' % (options.pattern, )) dbcurs.close() dbconn.close() + + +def do_romident(options): + dbconn = dbaccess.QueryConnection(options.database) + dbcurs = dbconn.cursor() + ident = _Identifier(dbcurs) + for path in options.path: + ident.processPath(path) + ident.printResults() + dbcurs.close() + dbconn.close() diff --git a/scripts/minimaws/lib/dbaccess.py b/scripts/minimaws/lib/dbaccess.py index 57de6aeb236..74c1cdb6d1e 100644 --- a/scripts/minimaws/lib/dbaccess.py +++ b/scripts/minimaws/lib/dbaccess.py @@ -1,21 +1,107 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb import sqlite3 -import sys - -if sys.version_info >= (3, 4): - import urllib.request +import urllib.request class SchemaQueries(object): + # software lists + CREATE_SOFTWAREINFOTYPE = \ + 'CREATE TABLE softwareinfotype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' name TEXT NOT NULL,\n' \ + ' UNIQUE (name ASC))' + CREATE_SOFTWARESHAREDFEATTYPE = \ + 'CREATE TABLE softwaresharedfeattype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' name TEXT NOT NULL,\n' \ + ' UNIQUE (name ASC))' + CREATE_SOFTWAREPARTFEATURETYPE = \ + 'CREATE TABLE softwarepartfeaturetype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' name TEXT NOT NULL,\n' \ + ' UNIQUE (name ASC))' + CREATE_SOFTWARELIST = \ + 'CREATE TABLE softwarelist (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' shortname TEXT NOT NULL,\n' \ + ' description TEXT NOT NULL,\n' \ + ' UNIQUE (shortname ASC))' + CREATE_SOFTWARELISTNOTES = \ + 'CREATE TABLE softwarelistnotes (\n' \ + ' softwarelist INTEGER PRIMARY KEY,\n' \ + ' notes TEXT NOT NULL,\n' \ + ' FOREIGN KEY (softwarelist) REFERENCES softwarelist (id))' + CREATE_SOFTWARE = \ + 'CREATE TABLE software (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' softwarelist INTEGER NOT NULL,\n' \ + ' shortname TEXT NOT NULL,\n' \ + ' supported INTEGER NOT NULL,\n' \ + ' description TEXT NOT NULL,\n' \ + ' year TEXT NOT NULL,\n' \ + ' publisher TEXT NOT NULL,\n' \ + ' UNIQUE (softwarelist ASC, shortname ASC),\n' \ + ' FOREIGN KEY (softwarelist) REFERENCES softwarelist (id))' + CREATE_SOFTWARECLONEOF = \ + 'CREATE TABLE softwarecloneof (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' parent INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (id) REFERENCES software (id),\n' \ + ' FOREIGN KEY (parent) REFERENCES software (id))' + CREATE_SOFTWARENOTES = \ + 'CREATE TABLE softwarenotes (\n' \ + ' software INTEGER PRIMARY KEY,\n' \ + ' notes TEXT NOT NULL,\n' \ + ' FOREIGN KEY (software) REFERENCES software (id))' + CREATE_SOFTWAREINFO = \ + 'CREATE TABLE softwareinfo (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' software INTEGER NOT NULL,\n' \ + ' infotype INTEGER NOT NULL,\n' \ + ' value TEXT NOT NULL,\n' \ + ' FOREIGN KEY (software) REFERENCES software (id),\n' \ + ' FOREIGN KEY (infotype) REFERENCES softwareinfotype (id))' + CREATE_SOFTWARESHAREDFEAT = \ + 'CREATE TABLE softwaresharedfeat (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' software INTEGER NOT NULL,\n' \ + ' sharedfeattype INTEGER NOT NULL,\n' \ + ' value TEXT NOT NULL,\n' \ + ' UNIQUE (software ASC, sharedfeattype ASC),\n' \ + ' FOREIGN KEY (software) REFERENCES software (id),\n' \ + ' FOREIGN KEY (sharedfeattype) REFERENCES softwaresharedfeattype (id))' + CREATE_SOFTWAREPART = \ + 'CREATE TABLE softwarepart (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' software INTEGER NOT NULL,\n' \ + ' shortname TEXT NOT NULL,\n' \ + ' interface TEXT NOT NULL,\n' \ + ' UNIQUE (software ASC, shortname ASC))' + CREATE_SOFTWAREPARTFEATURE = \ + 'CREATE TABLE softwarepartfeature (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' part INTEGER NOT NULL,\n' \ + ' featuretype INTEGER NOT NULL,\n' \ + ' value TEXT NOT NULL,\n' \ + ' UNIQUE (part ASC, featuretype ASC),\n' \ + ' FOREIGN KEY (part) REFERENCES softwarepart (id),\n' \ + ' FOREIGN KEY (featuretype) REFERENCES softwarepartfeaturetype (id))' + + # machines CREATE_FEATURETYPE = \ 'CREATE TABLE featuretype (\n' \ ' id INTEGER PRIMARY KEY,\n' \ ' name TEXT NOT NULL,\n' \ ' UNIQUE (name ASC))' + CREATE_MACHINESOFTWARELISTSTATUSTYPE = \ + 'CREATE TABLE machinesoftwareliststatustype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' value TEXT NOT NULL,\n' \ + ' UNIQUE (value ASC))' CREATE_SOURCEFILE = \ 'CREATE TABLE sourcefile (\n' \ ' id INTEGER PRIMARY KEY,\n' \ @@ -141,15 +227,97 @@ class SchemaQueries(object): ' size INTEGER NOT NULL,\n' \ ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ ' FOREIGN KEY (machine, size) REFERENCES ramoption (machine, size))' + CREATE_MACHINESOFTWARELIST = \ + 'CREATE TABLE machinesoftwarelist (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' softwarelist INTEGER NOT NULL,\n' \ + ' tag TEXT NOT NULL,\n' \ + ' status INTEGER NOT NULL,\n' \ + ' UNIQUE (machine ASC, tag ASC),\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (softwarelist) REFERENCES softwarelist (id),' \ + ' FOREIGN KEY (status) REFERENCES machinesoftwareliststatustype (id))' + # media + CREATE_ROM = \ + 'CREATE TABLE rom (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' crc INTEGER NOT NULL,\n' \ + ' sha1 TEXT NOT NULL,\n' \ + ' UNIQUE (crc ASC, sha1 ASC))' + CREATE_ROMDUMP = \ + 'CREATE TABLE romdump (\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' rom INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (rom) REFERENCES rom (id),\n' \ + ' UNIQUE (machine, rom, name))' + CREATE_SOFTWAREROMDUMP = \ + 'CREATE TABLE softwareromdump (\n' \ + ' part INTEGER NOT NULL,\n' \ + ' rom INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (part) REFERENCES softwarepart (id),\n' \ + ' FOREIGN KEY (rom) REFERENCES rom (id),\n' \ + ' UNIQUE (part, rom, name))' + CREATE_DISK = \ + 'CREATE TABLE disk (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' sha1 TEXT NOT NULL,\n' \ + ' UNIQUE (sha1 ASC))' + CREATE_DISKDUMP = \ + 'CREATE TABLE diskdump (\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' disk INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (disk) REFERENCES disk (id),\n' \ + ' UNIQUE (machine, disk, name))' + CREATE_SOFTWAREDISKDUMP = \ + 'CREATE TABLE softwarediskdump (\n' \ + ' part INTEGER NOT NULL,\n' \ + ' disk INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (part) REFERENCES softwarepart (id),\n' \ + ' FOREIGN KEY (disk) REFERENCES disk (id),\n' \ + ' UNIQUE (part, disk, name))' + + # create temporary tables used during loading + CREATE_TEMPORARY_SOFTWARECLONEOF = 'CREATE TEMPORARY TABLE temp_softwarecloneof (id INTEGER PRIMARY KEY, parent TEXT NOT NULL)' CREATE_TEMPORARY_DEVICEREFERENCE = 'CREATE TEMPORARY TABLE temp_devicereference (id INTEGER PRIMARY KEY, machine INTEGER NOT NULL, device TEXT NOT NULL, UNIQUE (machine, device))' CREATE_TEMPORARY_SLOTOPTION = 'CREATE TEMPORARY TABLE temp_slotoption (id INTEGER PRIMARY KEY, slot INTEGER NOT NULL, device TEXT NOT NULL, name TEXT NOT NULL)' CREATE_TEMPORARY_SLOTDEFAULT = 'CREATE TEMPORARY TABLE temp_slotdefault (id INTEGER PRIMARY KEY, slotoption INTEGER NOT NULL)' + # drop temporary tables used during loading + DROP_TEMPORARY_SOFTWARECLONEOF = 'DROP TABLE IF EXISTS temp_softwarecloneof' DROP_TEMPORARY_DEVICEREFERENCE = 'DROP TABLE IF EXISTS temp_devicereference' DROP_TEMPORARY_SLOTOPTION = 'DROP TABLE IF EXISTS temp_slotoption' DROP_TEMPORARY_SLOTDEFAULT = 'DROP TABLE IF EXISTS temp_slotdefault' + INDEX_SOFTWARE_SOFTWARELIST_SUPPORTED = 'CREATE INDEX software_softwarelist_supported ON software (softwarelist ASC, supported ASC)' + INDEX_SOFTWARE_SOFTWARELIST_YEAR = 'CREATE INDEX software_softwarelist_year ON software (softwarelist ASC, year ASC)' + INDEX_SOFTWARE_SOFTWARELIST_PUBLISHER = 'CREATE INDEX software_softwarelist_publisher ON software (softwarelist ASC, publisher ASC)' + INDEX_SOFTWARE_SHORTNAME_SOFTWARELIST = 'CREATE INDEX software_shortname_softwarelist ON software (shortname ASC, softwarelist ASC)' + INDEX_SOFTWARE_YEAR_SOFTWARELIST = 'CREATE INDEX software_year_softwarelist ON software (year ASC, softwarelist ASC)' + INDEX_SOFTWARE_PUBLISHER_SOFTWARELIST = 'CREATE INDEX software_publisher_softwarelist ON software (publisher ASC, softwarelist ASC)' + + INDEX_SOFTWARECLONEOF_PARENT = 'CREATE INDEX softwarecloneof_parent ON softwarecloneof (parent ASC)' + + INDEX_SOFTWAREINFO_SOFTWARE_INFOTYPE = 'CREATE INDEX softwareinfo_software_infotype ON softwareinfo (software ASC, infotype ASC)' + INDEX_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE = 'CREATE INDEX softwareinfo_infotype_value_software ON softwareinfo (infotype ASC, value ASC, software ASC)' + + INDEX_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE = 'CREATE INDEX softwaresharedfeat_sharedfeattype_value_software ON softwaresharedfeat (sharedfeattype ASC, value ASC, software ASC)' + + INDEX_SOFTWAREPART_INTERFACE_SOFTWARE = 'CREATE INDEX softwarepart_interface_software ON softwarepart (interface ASC, software ASC)' + + INDEX_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART = 'CREATE INDEX softwarepartfeature_featuretype_value_part ON softwarepartfeature (featuretype ASC, value ASC, part ASC)' + INDEX_MACHINE_ISDEVICE_SHORTNAME = 'CREATE INDEX machine_isdevice_shortname ON machine (isdevice ASC, shortname ASC)' INDEX_MACHINE_ISDEVICE_DESCRIPTION = 'CREATE INDEX machine_isdevice_description ON machine (isdevice ASC, description ASC)' INDEX_MACHINE_RUNNABLE_SHORTNAME = 'CREATE INDEX machine_runnable_shortname ON machine (runnable ASC, shortname ASC)' @@ -164,6 +332,38 @@ class SchemaQueries(object): INDEX_DIPSWITCH_MACHINE_ISCONFIG = 'CREATE INDEX dipswitch_machine_isconfig ON dipswitch (machine ASC, isconfig ASC)' + INDEX_MACHINESOFTWARELIST_SOFTWARELIST_MACHINE = 'CREATE INDEX machinesoftwarelist_softwarelist_machine ON machinesoftwarelist (softwarelist ASC, machine ASC)' + + INDEX_ROMDUMP_ROM = 'CREATE INDEX romdump_rom ON romdump (rom ASC)' + INDEX_ROMDUMP_MACHINE_BAD = 'CREATE INDEX romdump_machine_bad ON romdump (machine ASC, bad ASC)' + + INDEX_SOFTWAREROMDUMP_ROM = 'CREATE INDEX softwareromdump_rom ON softwareromdump (rom ASC)' + INDEX_SOFTWAREROMDUMP_PART_BAD = 'CREATE INDEX softwareromdump_part_bad ON softwareromdump (part ASC, bad ASC)' + + INDEX_DISKDUMP_DISK = 'CREATE INDEX diskdump_disk ON diskdump (disk ASC)' + INDEX_DISKDUMP_MACHINE_BAD = 'CREATE INDEX diskdump_machine_bad ON diskdump (machine ASC, bad ASC)' + + INDEX_SOFTWAREDISKDUMP_DISK = 'CREATE INDEX softwarediskdump_disk ON softwarediskdump (disk ASC)' + INDEX_SOFTWAREDISKDUMP_PART_BAD = 'CREATE INDEX softwarediskdump_part_bad ON softwarediskdump (part ASC, bad ASC)' + + DROP_SOFTWARE_SOFTWARELIST_SUPPORTED = 'DROP INDEX IF EXISTS software_softwarelist_supported' + DROP_SOFTWARE_SOFTWARELIST_YEAR = 'DROP INDEX IF EXISTS software_softwarelist_year' + DROP_SOFTWARE_SOFTWARELIST_PUBLISHER = 'DROP INDEX IF EXISTS software_softwarelist_publisher' + DROP_SOFTWARE_SHORTNAME_SOFTWARELIST = 'DROP INDEX IF EXISTS software_shortname_softwarelist' + DROP_SOFTWARE_YEAR_SOFTWARELIST = 'DROP INDEX IF EXISTS software_year_softwarelist' + DROP_SOFTWARE_PUBLISHER_SOFTWARELIST = 'DROP INDEX IF EXISTS software_publisher_softwarelist' + + DROP_SOFTWARECLONEOF_PARENT = 'DROP INDEX IF EXISTS softwarecloneof_parent' + + DROP_SOFTWAREINFO_SOFTWARE_INFOTYPE = 'DROP INDEX IF EXISTS softwareinfo_software_infotype' + DROP_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE = 'DROP INDEX IF EXISTS softwareinfo_infotype_value_software' + + DROP_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE = 'DROP INDEX IF EXISTS softwaresharedfeat_sharedfeattype_value_software' + + DROP_SOFTWAREPART_INTERFACE_SOFTWARE = 'DROP INDEX IF EXISTS softwarepart_interface_software' + + DROP_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART = 'DROP INDEX IF EXISTS softwarepartfeature_featuretype_value_part' + DROP_MACHINE_ISDEVICE_SHORTNAME = 'DROP INDEX IF EXISTS machine_isdevice_shortname' DROP_MACHINE_ISDEVICE_DESCRIPTION = 'DROP INDEX IF EXISTS machine_isdevice_description' DROP_MACHINE_RUNNABLE_SHORTNAME = 'DROP INDEX IF EXISTS machine_runnable_shortname' @@ -178,8 +378,35 @@ class SchemaQueries(object): DROP_DIPSWITCH_MACHINE_ISCONFIG = 'DROP INDEX IF EXISTS dipswitch_machine_isconfig' + DROP_MACHINESOFTWARELIST_SOFTWARELIST_MACHINE = 'DROP INDEX IF EXISTS machinesoftwarelist_softwarelist_machine' + + DROP_ROMDUMP_ROM = 'DROP INDEX IF EXISTS romdump_rom' + DROP_ROMDUMP_MACHINE_BAD = 'DROP INDEX IF EXISTS romdump_machine_bad' + + DROP_SOFTWAREROMDUMP_ROM = 'DROP INDEX IF EXISTS softwareromdump_rom' + DROP_SOFTWAREROMDUMP_PART_BAD = 'DROP INDEX IF EXISTS softwareromdump_part_bad' + + DROP_DISKDUMP_DISK = 'DROP INDEX IF EXISTS diskdump_disk' + DROP_DISKDUMP_MACHINE_BAD = 'DROP INDEX IF EXISTS diskdump_machine_bad' + + DROP_SOFTWAREDISKDUMP_DISK = 'DROP INDEX IF EXISTS softwarediskdump_disk' + DROP_SOFTWAREDISKDUMP_PART_BAD = 'DROP INDEX IF EXISTS softwarediskdump_part_bad' + CREATE_TABLES = ( + CREATE_SOFTWAREINFOTYPE, + CREATE_SOFTWARESHAREDFEATTYPE, + CREATE_SOFTWAREPARTFEATURETYPE, + CREATE_SOFTWARELIST, + CREATE_SOFTWARELISTNOTES, + CREATE_SOFTWARE, + CREATE_SOFTWARECLONEOF, + CREATE_SOFTWARENOTES, + CREATE_SOFTWAREINFO, + CREATE_SOFTWARESHAREDFEAT, + CREATE_SOFTWAREPART, + CREATE_SOFTWAREPARTFEATURE, CREATE_FEATURETYPE, + CREATE_MACHINESOFTWARELISTSTATUSTYPE, CREATE_SOURCEFILE, CREATE_MACHINE, CREATE_SYSTEM, @@ -196,14 +423,34 @@ class SchemaQueries(object): CREATE_SLOTOPTION, CREATE_SLOTDEFAULT, CREATE_RAMOPTION, - CREATE_RAMDEFAULT) + CREATE_RAMDEFAULT, + CREATE_MACHINESOFTWARELIST, + CREATE_ROM, + CREATE_ROMDUMP, + CREATE_SOFTWAREROMDUMP, + CREATE_DISK, + CREATE_DISKDUMP, + CREATE_SOFTWAREDISKDUMP) CREATE_TEMPORARY_TABLES = ( + CREATE_TEMPORARY_SOFTWARECLONEOF, CREATE_TEMPORARY_DEVICEREFERENCE, CREATE_TEMPORARY_SLOTOPTION, CREATE_TEMPORARY_SLOTDEFAULT) CREATE_INDEXES = ( + INDEX_SOFTWARE_SOFTWARELIST_SUPPORTED, + INDEX_SOFTWARE_SOFTWARELIST_YEAR, + INDEX_SOFTWARE_SOFTWARELIST_PUBLISHER, + INDEX_SOFTWARE_SHORTNAME_SOFTWARELIST, + INDEX_SOFTWARE_YEAR_SOFTWARELIST, + INDEX_SOFTWARE_PUBLISHER_SOFTWARELIST, + INDEX_SOFTWARECLONEOF_PARENT, + INDEX_SOFTWAREINFO_SOFTWARE_INFOTYPE, + INDEX_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE, + INDEX_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE, + INDEX_SOFTWAREPART_INTERFACE_SOFTWARE, + INDEX_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART, INDEX_MACHINE_ISDEVICE_SHORTNAME, INDEX_MACHINE_ISDEVICE_DESCRIPTION, INDEX_MACHINE_RUNNABLE_SHORTNAME, @@ -212,9 +459,30 @@ class SchemaQueries(object): INDEX_SYSTEM_MANUFACTURER, INDEX_ROMOF_PARENT, INDEX_CLONEOF_PARENT, - INDEX_DIPSWITCH_MACHINE_ISCONFIG) + INDEX_DIPSWITCH_MACHINE_ISCONFIG, + INDEX_MACHINESOFTWARELIST_SOFTWARELIST_MACHINE, + INDEX_ROMDUMP_ROM, + INDEX_ROMDUMP_MACHINE_BAD, + INDEX_SOFTWAREROMDUMP_ROM, + INDEX_SOFTWAREROMDUMP_PART_BAD, + INDEX_DISKDUMP_DISK, + INDEX_DISKDUMP_MACHINE_BAD, + INDEX_SOFTWAREDISKDUMP_DISK, + INDEX_SOFTWAREDISKDUMP_PART_BAD) DROP_INDEXES = ( + DROP_SOFTWARE_SOFTWARELIST_SUPPORTED, + DROP_SOFTWARE_SOFTWARELIST_YEAR, + DROP_SOFTWARE_SOFTWARELIST_PUBLISHER, + DROP_SOFTWARE_SHORTNAME_SOFTWARELIST, + DROP_SOFTWARE_YEAR_SOFTWARELIST, + DROP_SOFTWARE_PUBLISHER_SOFTWARELIST, + DROP_SOFTWARECLONEOF_PARENT, + DROP_SOFTWAREINFO_SOFTWARE_INFOTYPE, + DROP_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE, + DROP_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE, + DROP_SOFTWAREPART_INTERFACE_SOFTWARE, + DROP_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART, DROP_MACHINE_ISDEVICE_SHORTNAME, DROP_MACHINE_ISDEVICE_DESCRIPTION, DROP_MACHINE_RUNNABLE_SHORTNAME, @@ -223,11 +491,35 @@ class SchemaQueries(object): DROP_SYSTEM_MANUFACTURER, DROP_ROMOF_PARENT, DROP_CLONEOF_PARENT, - DROP_DIPSWITCH_MACHINE_ISCONFIG) + DROP_DIPSWITCH_MACHINE_ISCONFIG, + DROP_MACHINESOFTWARELIST_SOFTWARELIST_MACHINE, + DROP_ROMDUMP_ROM, + DROP_ROMDUMP_MACHINE_BAD, + DROP_SOFTWAREROMDUMP_ROM, + DROP_SOFTWAREROMDUMP_PART_BAD, + DROP_DISKDUMP_DISK, + DROP_DISKDUMP_MACHINE_BAD, + DROP_SOFTWAREDISKDUMP_DISK, + DROP_SOFTWAREDISKDUMP_PART_BAD) class UpdateQueries(object): + # software lists + ADD_SOFTWAREINFOTYPE = 'INSERT OR IGNORE INTO softwareinfotype (name) VALUES (?)' + ADD_SOFTWARESHAREDFEATTYPE = 'INSERT OR IGNORE INTO softwaresharedfeattype (name) VALUES (?)' + ADD_SOFTWAREPARTFEATURETYPE = 'INSERT OR IGNORE INTO softwarepartfeaturetype (name) VALUES(?)' + ADD_SOFTWARELIST = 'INSERT INTO softwarelist (shortname, description) VALUES (?, ?)' + ADD_SOFTWARELISTNOTES = 'INSERT INTO softwarelistnotes (softwarelist, notes) VALUES (?, ?)' + ADD_SOFTWARE = 'INSERT INTO software (softwarelist, shortname, supported, description, year, publisher) VALUES (?, ?, ?, ?, ?, ?)' + ADD_SOFTWARENOTES = 'INSERT INTO softwarenotes (software, notes) VALUES (?, ?)' + ADD_SOFTWAREINFO = 'INSERT INTO softwareinfo (software, infotype, value) SELECT ?, id, ? FROM softwareinfotype WHERE name = ?' + ADD_SOFTWARESHAREDFEAT = 'INSERT INTO softwaresharedfeat (software, sharedfeattype, value) SELECT ?, id, ? FROM softwaresharedfeattype WHERE name = ?' + ADD_SOFTWAREPART = 'INSERT INTO softwarepart (software, shortname, interface) VALUES (?, ?, ?)' + ADD_SOFTWAREPARTFEATURE = 'INSERT INTO softwarepartfeature (part, featuretype, value) SELECT ?, id, ? FROM softwarepartfeaturetype WHERE name = ?' + + # machines ADD_FEATURETYPE = 'INSERT OR IGNORE INTO featuretype (name) VALUES (?)' + ADD_MACHINESOFTWARELISTSTATUSTYPE = 'INSERT OR IGNORE INTO machinesoftwareliststatustype (value) VALUES (?)' ADD_SOURCEFILE = 'INSERT OR IGNORE INTO sourcefile (filename) VALUES (?)' ADD_MACHINE = 'INSERT INTO machine (shortname, description, sourcefile, isdevice, runnable) SELECT ?, ?, id, ?, ? FROM sourcefile WHERE filename = ?' ADD_SYSTEM = 'INSERT INTO system (id, year, manufacturer) VALUES (?, ?, ?)' @@ -242,19 +534,44 @@ class UpdateQueries(object): ADD_SLOT = 'INSERT INTO slot (machine, name) VALUES (?, ?)' ADD_RAMOPTION = 'INSERT INTO ramoption (machine, size, name) VALUES (?, ?, ?)' ADD_RAMDEFAULT = 'INSERT INTO ramdefault (machine, size) VALUES (?, ?)' - + ADD_MACHINESOFTWARELIST = 'INSERT INTO machinesoftwarelist (machine, softwarelist, tag, status) SELECT ?, softwarelist.id, ?, machinesoftwareliststatustype.id FROM softwarelist JOIN machinesoftwareliststatustype WHERE softwarelist.shortname = ? AND machinesoftwareliststatustype.value = ?' + + # media + ADD_ROM = 'INSERT OR IGNORE INTO rom (crc, sha1) VALUES (?, ?)' + ADD_ROMDUMP = 'INSERT OR IGNORE INTO romdump (machine, rom, name, bad) SELECT ?, id, ?, ? FROM rom WHERE crc = ? AND sha1 = ?' + ADD_SOFTWAREROMDUMP = 'INSERT OR IGNORE INTO softwareromdump (part, rom, name, bad) SELECT ?, id, ?, ? FROM rom WHERE crc = ? AND sha1 = ?' + ADD_DISK = 'INSERT OR IGNORE INTO disk (sha1) VALUES (?)' + ADD_DISKDUMP = 'INSERT OR IGNORE INTO diskdump (machine, disk, name, bad) SELECT ?, id, ?, ? FROM disk WHERE sha1 = ?' + ADD_SOFTWAREDISKDUMP = 'INSERT OR IGNORE INTO softwarediskdump (part, disk, name, bad) SELECT ?, id, ?, ? FROM disk WHERE sha1 = ?' + + # temporary records used during loading + ADD_TEMPORARY_SOFTWARECLONEOF = 'INSERT INTO temp_softwarecloneof (id, parent) VALUES (?, ?)' ADD_TEMPORARY_DEVICEREFERENCE = 'INSERT OR IGNORE INTO temp_devicereference (machine, device) VALUES (?, ?)' ADD_TEMPORARY_SLOTOPTION = 'INSERT INTO temp_slotoption (slot, device, name) VALUES (?, ?, ?)' ADD_TEMPORARY_SLOTDEFAULT = 'INSERT INTO temp_slotdefault (id, slotoption) VALUES (?, ?)' - FINALISE_DEVICEREFERENCES = 'INSERT INTO devicereference (id, machine, device) SELECT temp_devicereference.id, temp_devicereference.machine, machine.id FROM temp_devicereference LEFT JOIN machine ON temp_devicereference.device = machine.shortname' - FINALISE_SLOTOPTIONS = 'INSERT INTO slotoption (id, slot, device, name) SELECT temp_slotoption.id, temp_slotoption.slot, machine.id, temp_slotoption.name FROM temp_slotoption LEFT JOIN machine ON temp_slotoption.device = machine.shortname' - FINALISE_SLOTDEFAULTS = 'INSERT INTO slotdefault (id, slotoption) SELECT id, slotoption FROM temp_slotdefault' + # convert temporary tables to final form + FINALISE_SOFTWARECLONEOFS = \ + 'INSERT INTO softwarecloneof (id, parent) ' \ + 'SELECT temp_softwarecloneof.id AS id, parent.id AS parent ' \ + 'FROM temp_softwarecloneof LEFT JOIN software AS clone ON temp_softwarecloneof.id = clone.id LEFT JOIN software AS parent ON clone.softwarelist = parent.softwarelist AND temp_softwarecloneof.parent = parent.shortname' + FINALISE_DEVICEREFERENCES = \ + 'INSERT INTO devicereference (id, machine, device) ' \ + 'SELECT temp_devicereference.id, temp_devicereference.machine, machine.id ' \ + 'FROM temp_devicereference LEFT JOIN machine ON temp_devicereference.device = machine.shortname' + FINALISE_SLOTOPTIONS = \ + 'INSERT INTO slotoption (id, slot, device, name) ' \ + 'SELECT temp_slotoption.id, temp_slotoption.slot, machine.id, temp_slotoption.name ' \ + 'FROM temp_slotoption LEFT JOIN machine ON temp_slotoption.device = machine.shortname' + FINALISE_SLOTDEFAULTS = \ + 'INSERT INTO slotdefault (id, slotoption) ' \ + 'SELECT id, slotoption ' \ + 'FROM temp_slotdefault' class QueryCursor(object): def __init__(self, dbconn, **kwargs): - super(QueryCursor, self).__init__(**kwargs) + super().__init__(**kwargs) self.dbcurs = dbconn.cursor() def close(self): @@ -281,13 +598,13 @@ class QueryCursor(object): if pattern is not None: return self.dbcurs.execute( 'SELECT shortname, description ' \ - 'FROM machine WHERE isdevice = 0 AND shortname GLOB ? ' \ + 'FROM machine WHERE shortname GLOB ? ' \ 'ORDER BY shortname ASC', (pattern, )) else: return self.dbcurs.execute( 'SELECT shortname, description ' \ - 'FROM machine WHERE isdevice = 0 ' \ + 'FROM machine ' \ 'ORDER BY shortname ASC') def listsource(self, pattern): @@ -295,14 +612,14 @@ class QueryCursor(object): return self.dbcurs.execute( 'SELECT machine.shortname, sourcefile.filename ' \ 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id ' \ - 'WHERE machine.isdevice = 0 AND machine.shortname GLOB ? ' \ + 'WHERE machine.shortname GLOB ? ' \ 'ORDER BY machine.shortname ASC', (pattern, )) else: return self.dbcurs.execute( 'SELECT machine.shortname, sourcefile.filename ' \ 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id ' \ - 'WHERE machine.isdevice = 0 ORDER BY machine.shortname ASC') + 'ORDER BY machine.shortname ASC') def listclones(self, pattern): if pattern is not None: @@ -359,10 +676,10 @@ class QueryCursor(object): def get_machine_id(self, machine): return (self.dbcurs.execute('SELECT id FROM machine WHERE shortname = ?', (machine, )).fetchone() or (None, ))[0] - def get_machine_info(self, machine): + def get_machine_details(self, machine): return self.dbcurs.execute( 'SELECT machine.id AS id, machine.description AS description, machine.isdevice AS isdevice, machine.runnable AS runnable, sourcefile.filename AS sourcefile, system.year AS year, system.manufacturer AS manufacturer, cloneof.parent AS cloneof, romof.parent AS romof ' \ - 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id LEFT JOIN system ON machine.id = system.id LEFT JOIN cloneof ON system.id = cloneof.id LEFT JOIN romof ON system.id = romof.id ' \ + 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id LEFT JOIN system ON machine.id = system.id LEFT JOIN cloneof ON machine.id = cloneof.id LEFT JOIN romof ON machine.id = romof.id ' \ 'WHERE machine.shortname = ?', (machine, )) @@ -401,7 +718,7 @@ class QueryCursor(object): def get_sourcefile_machines(self, id): return self.dbcurs.execute( 'SELECT machine.shortname AS shortname, machine.description AS description, machine.isdevice AS isdevice, machine.runnable AS runnable, sourcefile.filename AS sourcefile, system.year AS year, system.manufacturer AS manufacturer, cloneof.parent AS cloneof, romof.parent AS romof ' \ - 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id LEFT JOIN system ON machine.id = system.id LEFT JOIN cloneof ON system.id = cloneof.id LEFT JOIN romof ON system.id = romof.id ' \ + 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id LEFT JOIN system ON machine.id = system.id LEFT JOIN cloneof ON machine.id = cloneof.id LEFT JOIN romof ON machine.id = romof.id ' \ 'WHERE machine.sourcefile = ?', (id, )) @@ -425,6 +742,13 @@ class QueryCursor(object): else: return self.dbcurs.execute('SELECT COUNT(*) FROM sourcefile').fetchone()[0] + def get_clones(self, shortname): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, system.year AS year, system.manufacturer AS manufacturer ' \ + 'FROM cloneof JOIN machine ON cloneof.id = machine.id LEFT JOIN system ON machine.id = system.id ' \ + 'WHERE cloneof.parent = ?', + (shortname, )) + def count_slots(self, machine): return self.dbcurs.execute( 'SELECT COUNT(*) FROM slot WHERE machine = ?', (machine, )).fetchone()[0] @@ -458,15 +782,186 @@ class QueryCursor(object): 'ORDER BY ramoption.size', (machine, )) + def get_machine_softwarelists(self, machine): + return self.dbcurs.execute( + 'SELECT machinesoftwarelist.tag AS tag, machinesoftwareliststatustype.value AS status, softwarelist.shortname AS shortname, softwarelist.description AS description, COUNT(software.id) AS total, COUNT(CASE software.supported WHEN 0 THEN 1 ELSE NULL END) AS supported, COUNT(CASE software.supported WHEN 1 THEN 1 ELSE NULL END) AS partiallysupported, COUNT(CASE software.supported WHEN 2 THEN 1 ELSE NULL END) AS unsupported ' \ + 'FROM machinesoftwarelist LEFT JOIN machinesoftwareliststatustype ON machinesoftwarelist.status = machinesoftwareliststatustype.id LEFT JOIN softwarelist ON machinesoftwarelist.softwarelist = softwarelist.id LEFT JOIN software ON softwarelist.id = software.softwarelist ' \ + 'WHERE machinesoftwarelist.machine = ? ' \ + 'GROUP BY machinesoftwarelist.id', + (machine, )) + + def get_softwarelist_id(self, shortname): + return (self.dbcurs.execute('SELECT id FROM softwarelist WHERE shortname = ?', (shortname, )).fetchone() or (None, ))[0] + + def get_softwarelist_details(self, shortname, pattern): + if pattern is not None: + return self.dbcurs.execute( + 'SELECT softwarelist.id AS id, softwarelist.shortname AS shortname, softwarelist.description AS description, softwarelistnotes.notes AS notes, COUNT(software.id) AS total, COUNT(CASE software.supported WHEN 0 THEN 1 ELSE NULL END) AS supported, COUNT(CASE software.supported WHEN 1 THEN 1 ELSE NULL END) AS partiallysupported, COUNT(CASE software.supported WHEN 2 THEN 1 ELSE NULL END) AS unsupported ' \ + 'FROM softwarelist LEFT JOIN softwarelistnotes ON softwarelist.id = softwarelistnotes.softwarelist LEFT JOIN software ON softwarelist.id = software.softwarelist ' \ + 'WHERE softwarelist.shortname = ? AND software.shortname GLOB ? ' \ + 'GROUP BY softwarelist.id', + (shortname, pattern)) + else: + return self.dbcurs.execute( + 'SELECT softwarelist.id AS id, softwarelist.shortname AS shortname, softwarelist.description AS description, softwarelistnotes.notes AS notes, COUNT(software.id) AS total, COUNT(CASE software.supported WHEN 0 THEN 1 ELSE NULL END) AS supported, COUNT(CASE software.supported WHEN 1 THEN 1 ELSE NULL END) AS partiallysupported, COUNT(CASE software.supported WHEN 2 THEN 1 ELSE NULL END) AS unsupported ' \ + 'FROM softwarelist LEFT JOIN softwarelistnotes ON softwarelist.id = softwarelistnotes.softwarelist LEFT JOIN software ON softwarelist.id = software.softwarelist ' \ + 'WHERE softwarelist.shortname = ? ' \ + 'GROUP BY softwarelist.id', + (shortname, )) + + def get_softwarelist_software(self, id, pattern): + if pattern is not None: + return self.dbcurs.execute( + 'SELECT software.shortname AS shortname, software.description AS description, software.year AS year, software.publisher AS publisher, software.supported AS supported, COUNT(DISTINCT softwarepart.id) AS parts, COUNT(softwareromdump.rom) + COUNT(softwarediskdump.disk) AS baddumps, parent.shortname AS parent, parent.description AS parentdescription, softwarelist.shortname AS parentsoftwarelist, softwarelist.description AS parentsoftwarelistdescription ' \ + 'FROM software JOIN softwarepart ON software.id = softwarepart.software LEFT JOIN softwareromdump ON softwarepart.id = softwareromdump.part AND softwareromdump.bad = 1 LEFT JOIN softwarediskdump ON softwarepart.id = softwarediskdump.part AND softwarediskdump.bad = 1 LEFT JOIN softwarecloneof ON software.id = softwarecloneof.id LEFT JOIN software AS parent ON softwarecloneof.parent = parent.id LEFT JOIN softwarelist ON parent.softwarelist = softwarelist.id ' \ + 'WHERE software.softwarelist = ? AND software.shortname GLOB ? ' \ + 'GROUP BY software.id', + (id, pattern)) + else: + return self.dbcurs.execute( + 'SELECT software.shortname AS shortname, software.description AS description, software.year AS year, software.publisher AS publisher, software.supported AS supported, COUNT(DISTINCT softwarepart.id) AS parts, COUNT(softwareromdump.rom) + COUNT(softwarediskdump.disk) AS baddumps, parent.shortname AS parent, parent.description AS parentdescription, softwarelist.shortname AS parentsoftwarelist, softwarelist.description AS parentsoftwarelistdescription ' \ + 'FROM software JOIN softwarepart ON software.id = softwarepart.software LEFT JOIN softwareromdump ON softwarepart.id = softwareromdump.part AND softwareromdump.bad = 1 LEFT JOIN softwarediskdump ON softwarepart.id = softwarediskdump.part AND softwarediskdump.bad = 1 LEFT JOIN softwarecloneof ON software.id = softwarecloneof.id LEFT JOIN software AS parent ON softwarecloneof.parent = parent.id LEFT JOIN softwarelist ON parent.softwarelist = softwarelist.id ' \ + 'WHERE software.softwarelist = ? ' \ + 'GROUP BY software.id', + (id, )) + + def get_softwarelist_machines(self, id): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, system.year AS year, system.manufacturer AS manufacturer, machinesoftwareliststatustype.value AS status ' \ + 'FROM (SELECT DISTINCT machine, status FROM machinesoftwarelist WHERE softwarelist = ?) AS softwarelists LEFT JOIN machine ON softwarelists.machine = machine.id LEFT JOIN system ON machine.id = system.id LEFT JOIN machinesoftwareliststatustype ON softwarelists.status = machinesoftwareliststatustype.id', + (id, )) + + def get_softwarelists(self, pattern): + if pattern is not None: + return self.dbcurs.execute( + 'SELECT softwarelist.shortname AS shortname, softwarelist.description AS description, COUNT(software.id) AS total, COUNT(CASE software.supported WHEN 0 THEN 1 ELSE NULL END) AS supported, COUNT(CASE software.supported WHEN 1 THEN 1 ELSE NULL END) AS partiallysupported, COUNT(CASE software.supported WHEN 2 THEN 1 ELSE NULL END) AS unsupported ' \ + 'FROM softwarelist LEFT JOIN software ON softwarelist.id = software.softwarelist ' \ + 'WHERE softwarelist.shortname GLOB ? ' \ + 'GROUP BY softwarelist.id', + (pattern, )) + else: + return self.dbcurs.execute( + 'SELECT softwarelist.shortname AS shortname, softwarelist.description AS description, COUNT(software.id) AS total, COUNT(CASE software.supported WHEN 0 THEN 1 ELSE NULL END) AS supported, COUNT(CASE software.supported WHEN 1 THEN 1 ELSE NULL END) AS partiallysupported, COUNT(CASE software.supported WHEN 2 THEN 1 ELSE NULL END) AS unsupported ' \ + 'FROM softwarelist LEFT JOIN software ON softwarelist.id = software.softwarelist ' \ + 'GROUP BY softwarelist.id') + + def get_software_details(self, softwarelist, software): + return self.dbcurs.execute( + 'SELECT software.id AS id, software.shortname AS shortname, software.supported AS supported, software.description AS description, software.year AS year, software.publisher AS publisher, softwarelist.shortname AS softwarelist, softwarelist.description AS softwarelistdescription, parent.shortname AS parent, parent.description AS parentdescription, parentsoftwarelist.shortname AS parentsoftwarelist, parentsoftwarelist.description AS parentsoftwarelistdescription, softwarenotes.notes AS notes ' \ + 'FROM software LEFT JOIN softwarelist ON software.softwarelist = softwarelist.id LEFT JOIN softwarecloneof ON software.id = softwarecloneof.id LEFT JOIN software AS parent ON softwarecloneof.parent = parent.id LEFT JOIN softwarelist AS parentsoftwarelist ON parent.softwarelist = parentsoftwarelist.id LEFT JOIN softwarenotes ON softwarenotes.software = software.id ' \ + 'WHERE software.softwarelist = (SELECT id FROM softwarelist WHERE shortname = ?) AND software.shortname = ?', + (softwarelist, software)) + + def get_software_clones(self, software): + return self.dbcurs.execute( + 'SELECT software.shortname AS shortname, software.description AS description, software.year AS year, software.publisher AS publisher, software.supported AS supported, softwarelist.shortname AS softwarelist, softwarelist.description AS softwarelistdescription ' \ + 'FROM softwarecloneof LEFT JOIN software ON softwarecloneof.id = software.id LEFT JOIN softwarelist ON software.softwarelist = softwarelist.id ' \ + 'WHERE softwarecloneof.parent = ?', + (software, )) + + def get_software_info(self, software): + return self.dbcurs.execute( + 'SELECT softwareinfotype.name AS name, softwareinfo.value AS value ' \ + 'FROM softwareinfo JOIN softwareinfotype ON softwareinfo.infotype = softwareinfotype.id ' \ + 'WHERE softwareinfo.software = ? ' \ + 'ORDER BY softwareinfotype.name ASC, softwareinfo.value ASC', + (software, )) + + def get_software_parts(self, software): + return self.dbcurs.execute( + 'SELECT softwarepart.id AS id, softwarepart.shortname AS shortname, softwarepart.interface AS interface, softwarepartfeature.value AS part_id ' \ + 'FROM softwarepart LEFT JOIN softwarepartfeature ON softwarepart.id = softwarepartfeature.part AND softwarepartfeature.featuretype = (SELECT id FROM softwarepartfeaturetype WHERE name = \'part_id\') ' \ + 'WHERE softwarepart.software = ?', + (software, )) + + def get_softwarepart_features(self, part): + return self.dbcurs.execute( + 'SELECT softwarepartfeaturetype.name AS name, softwarepartfeature.value AS value ' \ + 'FROM softwarepartfeature LEFT JOIN softwarepartfeaturetype ON softwarepartfeature.featuretype = softwarepartfeaturetype.id ' \ + 'WHERE softwarepartfeature.part = ? ' + 'ORDER BY softwarepartfeaturetype.name ASC', + (part, )) + + def get_rom_dumps(self, crc, sha1): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, romdump.name AS label, romdump.bad AS bad ' \ + 'FROM romdump LEFT JOIN machine ON romdump.machine = machine.id ' \ + 'WHERE romdump.rom = (SELECT id FROM rom WHERE crc = ? AND sha1 = ?)', + (crc, sha1)) + + def get_software_rom_dumps(self, crc, sha1): + return self.dbcurs.execute( + 'SELECT softwarelist.shortname AS softwarelist, softwarelist.description AS softwarelistdescription, software.shortname AS shortname, software.description AS description, softwarepart.shortname AS part, softwarepartfeature.value AS part_id, softwareromdump.name AS label, softwareromdump.bad AS bad ' \ + 'FROM softwareromdump LEFT JOIN softwarepart ON softwareromdump.part = softwarepart.id LEFT JOIN softwarepartfeature ON softwarepart.id = softwarepartfeature.part AND softwarepartfeature.featuretype = (SELECT id FROM softwarepartfeaturetype WHERE name = \'part_id\') LEFT JOIN software ON softwarepart.software = software.id LEFT JOIN softwarelist ON software.softwarelist = softwarelist.id ' \ + 'WHERE softwareromdump.rom = (SELECT id FROM rom WHERE crc = ? AND sha1 = ?)', + (crc, sha1)) + + def get_disk_dumps(self, sha1): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, diskdump.name AS label, diskdump.bad AS bad ' \ + 'FROM diskdump LEFT JOIN machine ON diskdump.machine = machine.id ' \ + 'WHERE diskdump.disk = (SELECT id FROM disk WHERE sha1 = ?)', + (sha1, )) + + def get_software_disk_dumps(self, sha1): + return self.dbcurs.execute( + 'SELECT softwarelist.shortname AS softwarelist, softwarelist.description AS softwarelistdescription, software.shortname AS shortname, software.description AS description, softwarepart.shortname AS part, softwarepartfeature.value AS part_id, softwarediskdump.name AS label, softwarediskdump.bad AS bad ' \ + 'FROM softwarediskdump LEFT JOIN softwarepart ON softwarediskdump.part = softwarepart.id LEFT JOIN softwarepartfeature ON softwarepart.id = softwarepartfeature.part AND softwarepartfeature.featuretype = (SELECT id FROM softwarepartfeaturetype WHERE name = \'part_id\') LEFT JOIN software ON softwarepart.software = software.id LEFT JOIN softwarelist ON software.softwarelist = softwarelist.id ' \ + 'WHERE softwarediskdump.disk = (SELECT id FROM disk WHERE sha1 = ?)', + (sha1, )) + class UpdateCursor(object): def __init__(self, dbconn, **kwargs): - super(UpdateCursor, self).__init__(**kwargs) + super().__init__(**kwargs) self.dbcurs = dbconn.cursor() def close(self): self.dbcurs.close() + def add_softwarelist(self, shortname, description): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARELIST, (shortname, description)) + return self.dbcurs.lastrowid + + def add_softwarelistnotes(self, softwarelist, notes): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARELISTNOTES, (softwarelist, notes)) + + def add_softwareinfotype(self, name): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREINFOTYPE, (name, )) + + def add_softwaresharedfeattype(self, name): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARESHAREDFEATTYPE, (name, )) + + def add_softwarepartfeaturetype(self, name): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREPARTFEATURETYPE, (name, )) + + def add_software(self, softwarelist, shortname, supported, description, year, publisher): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARE, (softwarelist, shortname, supported, description, year, publisher)) + return self.dbcurs.lastrowid + + def add_softwarecloneof(self, software, parent): + self.dbcurs.execute(UpdateQueries.ADD_TEMPORARY_SOFTWARECLONEOF, (software, parent)) + return self.dbcurs.lastrowid + + def add_softwarenotes(self, software, notes): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARENOTES, (software, notes)) + + def add_softwareinfo(self, software, infotype, value): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREINFO, (software, value, infotype)) + return self.dbcurs.lastrowid + + def add_softwaresharedfeat(self, software, sharedfeattype, value): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARESHAREDFEAT, (software, value, sharedfeattype)) + return self.dbcurs.lastrowid + + def add_softwarepart(self, software, shortname, interface): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREPART, (software, shortname, interface)) + return self.dbcurs.lastrowid + + def add_softwarepartfeature(self, part, featuretype, value): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREPARTFEATURE, (part, value, featuretype)) + return self.dbcurs.lastrowid + def add_featuretype(self, name): self.dbcurs.execute(UpdateQueries.ADD_FEATURETYPE, (name, )) @@ -536,14 +1031,40 @@ class UpdateCursor(object): self.dbcurs.execute(UpdateQueries.ADD_RAMDEFAULT, (machine, size)) return self.dbcurs.lastrowid + def add_machinesoftwarelist(self, machine, softwarelist, tag, status): + self.dbcurs.execute(UpdateQueries.ADD_MACHINESOFTWARELISTSTATUSTYPE, (status, )) + self.dbcurs.execute(UpdateQueries.ADD_MACHINESOFTWARELIST, (machine, tag, softwarelist, status)) + return self.dbcurs.lastrowid + + def add_rom(self, crc, sha1): + self.dbcurs.execute(UpdateQueries.ADD_ROM, (crc, sha1)) + return self.dbcurs.lastrowid + + def add_romdump(self, machine, name, crc, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_ROMDUMP, (machine, name, 1 if bad else 0, crc, sha1)) + return self.dbcurs.lastrowid + + def add_softwareromdump(self, part, name, crc, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREROMDUMP, (part, name, 1 if bad else 0, crc, sha1)) + return self.dbcurs.lastrowid + + def add_disk(self, sha1): + self.dbcurs.execute(UpdateQueries.ADD_DISK, (sha1, )) + return self.dbcurs.lastrowid + + def add_diskdump(self, machine, name, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_DISKDUMP, (machine, name, 1 if bad else 0, sha1)) + return self.dbcurs.lastrowid + + def add_softwarediskdump(self, part, name, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREDISKDUMP, (part, name, 1 if bad else 0, sha1)) + return self.dbcurs.lastrowid + class QueryConnection(object): def __init__(self, database, **kwargs): - super(QueryConnection, self).__init__(**kwargs) - if sys.version_info >= (3, 4): - self.dbconn = sqlite3.connect('file:' + urllib.request.pathname2url(database) + '?mode=ro', uri=True) - else: - self.dbconn = sqlite3.connect(database) + super().__init__(**kwargs) + self.dbconn = sqlite3.connect('file:' + urllib.request.pathname2url(database) + '?mode=ro', uri=True, check_same_thread=False) self.dbconn.row_factory = sqlite3.Row self.dbconn.execute('PRAGMA foreign_keys = ON') @@ -556,7 +1077,7 @@ class QueryConnection(object): class UpdateConnection(object): def __init__(self, database, **kwargs): - super(UpdateConnection, self).__init__(**kwargs) + super().__init__(**kwargs) self.dbconn = sqlite3.connect(database) self.dbconn.execute('PRAGMA page_size = 4096') self.dbconn.execute('PRAGMA foreign_keys = ON') @@ -576,6 +1097,8 @@ class UpdateConnection(object): def prepare_for_load(self): # here be dragons - this is a poor man's DROP ALL TABLES etc. self.dbconn.execute('PRAGMA foreign_keys = OFF') + for query in self.dbconn.execute('SELECT \'DROP VIEW \' || name FROM sqlite_master WHERE type = \'view\'').fetchall(): + self.dbconn.execute(query[0]) for query in self.dbconn.execute('SELECT \'DROP INDEX \' || name FROM sqlite_master WHERE type = \'index\' AND NOT name GLOB \'sqlite_autoindex_*\'').fetchall(): self.dbconn.execute(query[0]) for query in self.dbconn.execute('SELECT \'DROP TABLE \' || name FROM sqlite_master WHERE type = \'table\'').fetchall(): @@ -592,6 +1115,9 @@ class UpdateConnection(object): self.dbconn.commit() def finalise_load(self): + self.dbconn.execute(UpdateQueries.FINALISE_SOFTWARECLONEOFS) + self.dbconn.commit() + self.dbconn.execute(SchemaQueries.DROP_TEMPORARY_SOFTWARECLONEOF) self.dbconn.execute(UpdateQueries.FINALISE_DEVICEREFERENCES) self.dbconn.commit() self.dbconn.execute(SchemaQueries.DROP_TEMPORARY_DEVICEREFERENCE) diff --git a/scripts/minimaws/lib/htmltmpl.py b/scripts/minimaws/lib/htmltmpl.py index 1a030d04e93..c6489e27349 100644 --- a/scripts/minimaws/lib/htmltmpl.py +++ b/scripts/minimaws/lib/htmltmpl.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb @@ -24,6 +24,7 @@ SORTABLE_TABLE_EPILOGUE = string.Template( '</table>\n' '<script>make_table_sortable(document.getElementById("${id}"));</script>\n') + MACHINE_PROLOGUE = string.Template( '<!DOCTYPE html>\n' \ '<html>\n' \ @@ -48,32 +49,104 @@ MACHINE_PROLOGUE = string.Template( ' <tr><th>Runnable:</th><td>${runnable}</td></tr>\n' \ ' <tr><th>Source file:</th><td><a href="${sourcehref}">${sourcefile}</a></td></tr>\n') +MACHINE_CLONES_PROLOGUE = string.Template( + '<h2 id="heading-clones">Clones</h2>\n' \ + '<table id="tbl-clones">\n' \ + ' <thead>\n' \ + ' <tr>\n' \ + ' <th>Short name</th>\n' \ + ' <th>Description</th>\n' \ + ' <th>Year</th>\n' \ + ' <th>Manufacturer</th>\n' \ + ' </tr>\n' \ + ' </thead>\n' \ + ' <tbody>\n') + +MACHINE_CLONES_ROW = string.Template( + ' <tr>\n' \ + ' <td><a href="${href}">${shortname}</a></td>\n' \ + ' <td><a href="${href}">${description}</a></td>\n' \ + ' <td>${year}</td>\n' \ + ' <td>${manufacturer}</td>\n' \ + ' </tr>\n') + +MACHINE_SOFTWARELISTS_TABLE_PROLOGUE = string.Template( + '<h2 id="heading-softwarelists">Software Lists</h2>\n' \ + '<table id="tbl-softwarelists">\n' \ + ' <thead>\n' \ + ' <tr>\n' \ + ' <th>Card</th>\n' \ + ' <th>Short name</th>\n' \ + ' <th>Description</th>\n' \ + ' <th>Status</th>\n' \ + ' <th class="numeric">Total</th>\n' \ + ' <th class="numeric">Supported</th>\n' \ + ' <th class="numeric">Partially supported</th>\n' \ + ' <th class="numeric">Unsupported</th>\n' \ + ' </tr>\n' \ + ' </thead>\n' \ + ' <tbody>\n') + +MACHINE_SOFTWARELISTS_TABLE_ROW = string.Template( + ' <tr id="row-softwarelists-${rowid}">\n' \ + ' <td></td>\n' \ + ' <td><a href="${href}">${shortname}</a></td>\n' \ + ' <td><a href="${href}">${description}</a></td>\n' \ + ' <td>${status}</td>\n' \ + ' <td style="text-align: right">${total}</td>\n' \ + ' <td style="text-align: right">${supported}</td>\n' \ + ' <td style="text-align: right">${partiallysupported}</td>\n' \ + ' <td style="text-align: right">${unsupported}</td>\n' \ + ' </tr>\n') + +MACHINE_SOFTWARELISTS_TABLE_EPILOGUE = string.Template( + ' </tbody>\n' \ + '</table>\n' \ + '<script>\n' \ + ' make_table_sortable(document.getElementById("tbl-softwarelists"));\n' \ + ' make_collapsible(document.getElementById("heading-softwarelists"), document.getElementById("tbl-softwarelists"));\n' \ + ' if (!document.getElementById("tbl-softwarelists").tBodies[0].rows.length)\n' \ + ' {\n' \ + ' document.getElementById("heading-softwarelists").style.display = "none";\n' \ + ' document.getElementById("tbl-softwarelists").style.display = "none";\n' \ + ' }\n' \ + '</script>\n') + MACHINE_OPTIONS_HEADING = string.Template( - '<h2>Options</h2>\n' \ - '<p>\n' \ - ' Format: <select id="select-options-format" onchange="update_cmd_preview()"><option value="cmd">Command line</option><option value="ini">INI file</option></select>\n' \ - ' <input type="checkbox" id="check-explicit-defaults" onchange="update_cmd_preview()"><label for="check-explicit-defaults">Explicit defaults</label>\n' \ - '</p>\n' \ - '<p id="para-cmd-preview"></p>\n') + '<h2 id="heading-options">Options</h2>\n' \ + '<div id="div-options">\n' \ + ' <p>\n' \ + ' Format: <select id="select-options-format" onchange="update_cmd_preview()"><option value="cmd">Command line</option><option value="ini">INI file</option></select>\n' \ + ' <input type="checkbox" id="check-explicit-defaults" onchange="update_cmd_preview()"><label for="check-explicit-defaults">Explicit defaults</label>\n' \ + ' </p>\n' \ + ' <p id="para-cmd-preview"></p>\n') + +MACHINE_OPTIONS_EPILOGUE = string.Template( + '</div>\n' \ + '<script>make_collapsible(document.getElementById("heading-options"), document.getElementById("div-options"));</script>\n') MACHINE_BIOS_PROLOGUE = string.Template( - '<h3>System BIOS</h3>' \ - '<select id="select-system-bios" onchange="update_cmd_preview()">') + ' <h3>System BIOS</h3>' \ + ' <select id="select-system-bios" onchange="update_cmd_preview()">') MACHINE_BIOS_OPTION = string.Template( - ' <option value="${name}" data-isdefault="${isdefault}">${name} - ${description}</option>\n') + ' <option value="${name}" data-isdefault="${isdefault}">${name} - ${description}</option>\n') MACHINE_RAM_PROLOGUE = string.Template( - '<h3>RAM Size</h3>' \ - '<select id="select-ram-option" onchange="update_cmd_preview()">') + ' <h3>RAM Size</h3>\n' \ + ' <select id="select-ram-option" onchange="update_cmd_preview()">\n') MACHINE_RAM_OPTION = string.Template( - ' <option value="${name}" data-isdefault="${isdefault}">${name} (${size})</option>\n') + ' <option value="${name}" data-isdefault="${isdefault}">${name} (${size})</option>\n') -MACHINE_SLOTS_PLACEHOLDER = string.Template( - '<h3>Slots</h3>\n' \ - '<p id="para-slots-placeholder">Loading slot information…<p>\n' \ - '<script>fetch_slots("${machine}");</script>\n') +MACHINE_SLOTS_PLACEHOLDER_PROLOGUE = string.Template( + ' <h3>Slots</h3>\n' \ + ' <p id="para-slots-placeholder">Loading slot information…<p>\n' \ + ' <script>\n') + +MACHINE_SLOTS_PLACEHOLDER_EPILOGUE = string.Template( + ' populate_slots(${machine});\n' + ' </script>\n') MACHINE_ROW = string.Template( ' <tr>\n' \ @@ -163,3 +236,242 @@ SOURCEFILE_LIST_ROW = string.Template( ' <td>${sourcefile}</td>\n' \ ' <td style="text-align: right">${machines}</td>\n' \ ' </tr>\n') + + +SOFTWARE_PROLOGUE = string.Template( + '<!DOCTYPE html>\n' \ + '<html>\n' \ + '<head>\n' \ + ' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n' \ + ' <meta http-equiv="Content-Style-Type" content="text/css">\n' \ + ' <meta http-equiv="Content-Script-Type" content="text/javascript">\n' \ + ' <link rel="stylesheet" type="text/css" href="${assets}/style.css">\n' \ + ' <script type="text/javascript">var assetsurl="${assets}"</script>\n' \ + ' <script type="text/javascript" src="${assets}/common.js"></script>\n' \ + ' <title>${title}</title>\n' \ + '</head>\n' \ + '<body>\n' \ + '<h1>${heading}</h1>\n' \ + '<table class="sysinfo">\n' \ + ' <tr><th>Software list:</th><td><a href="${softwarelisthref}">${softwarelistdescription} (${softwarelist})</a></td></tr>\n' \ + ' <tr><th>Short name:</th><td>${shortname}</td></tr>\n' \ + ' <tr><th>Year:</th><td>${year}</td></tr>\n' \ + ' <tr><th>Publisher:</th><td>${publisher}</td></tr>\n'); + +SOFTWARE_CLONES_PROLOGUE = string.Template( + '<h2 id="heading-clones">Clones</h2>\n' \ + '<table id="tbl-clones">\n' \ + ' <thead>\n' \ + ' <tr>\n' \ + ' <th>Short name</th>\n' \ + ' <th>Description</th>\n' \ + ' <th>Year</th>\n' \ + ' <th>Publisher</th>\n' \ + ' <th>Supported</th>\n' \ + ' </tr>\n' \ + ' </thead>\n' \ + ' <tbody>\n') + +SOFTWARE_CLONES_ROW = string.Template( + ' <tr>\n' \ + ' <td><a href="${href}">${shortname}</a></td>\n' \ + ' <td><a href="${href}">${description}</a></td>\n' \ + ' <td>${year}</td>\n' \ + ' <td>${publisher}</td>\n' \ + ' <td>${supported}</td>\n' \ + ' </tr>\n') + +SOFTWARE_NOTES_PROLOGUE = string.Template( + '<h2 id="heading-notes">Notes</h2>\n' \ + '<div id="div-notes">\n') + +SOFTWARE_NOTES_EPILOGUE = string.Template( + '</div>\n' \ + '<script>make_collapsible(document.getElementById("heading-notes"), document.getElementById("div-notes"))();</script>\n\n') + +SOFTWARE_PARTS_PROLOGUE = string.Template( + '<h2 id="heading-parts">Parts</h2>\n' \ + '<div id="div-parts">\n\n') + +SOFTWARE_PARTS_EPILOGUE = string.Template( + '</div>\n' \ + '<script>make_collapsible(document.getElementById("heading-parts"), document.getElementById("div-parts"));</script>\n\n') + +SOFTWARE_PART_PROLOGUE = string.Template( + ' <h3>${heading}</h3>\n' \ + ' <table class="sysinfo">\n' \ + ' <tr><th>Short name:</th><td>${shortname}</td></tr>\n' \ + ' <tr><th>Interface:</th><td>${interface}</td></tr>\n') + + +SOFTWARELIST_PROLOGUE = string.Template( + '<!DOCTYPE html>\n' \ + '<html>\n' \ + '<head>\n' \ + ' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n' \ + ' <meta http-equiv="Content-Style-Type" content="text/css">\n' \ + ' <meta http-equiv="Content-Script-Type" content="text/javascript">\n' \ + ' <link rel="stylesheet" type="text/css" href="${assets}/style.css">\n' \ + ' <script type="text/javascript">var assetsurl="${assets}"</script>\n' \ + ' <script type="text/javascript" src="${assets}/common.js"></script>\n' \ + ' <title>${title}</title>\n' \ + '</head>\n' \ + '<body>\n' \ + '<h1>${heading}</h1>\n' \ + '<table class="sysinfo">\n' \ + ' <tr>\n' \ + ' <th>Short name:</th>\n' \ + ' <td>${shortname}</td>\n' \ + ' </tr>\n' \ + ' <tr>\n' \ + ' <th>Total:</th>\n' \ + ' <td style="text-align: right">${total}</td>\n' \ + ' </tr>\n' \ + ' <tr>\n' \ + ' <th>Supported:</th>\n' \ + ' <td style="text-align: right">${supported}</td>\n' \ + ' <td style="text-align: right">(${supportedpc}%)</td>\n' \ + ' </tr>\n' \ + ' <tr>\n' \ + ' <th>Partially supported:</th>\n' \ + ' <td style="text-align: right">${partiallysupported}</td>\n' \ + ' <td style="text-align: right">(${partiallysupportedpc}%)</td>\n' \ + ' </tr>\n' \ + ' <tr>\n' \ + ' <th>Unsupported:</th>\n' \ + ' <td style="text-align: right">${unsupported}</td>\n' \ + ' <td style="text-align: right">(${unsupportedpc}%)</td>\n' \ + ' </tr>\n' \ + '</table>\n') + +SOFTWARELIST_NOTES_PROLOGUE = string.Template( + '<h2 id="heading-notes">Notes</h2>\n' \ + '<div id="div-notes">\n') + +SOFTWARELIST_NOTES_EPILOGUE = string.Template( + '</div>\n' \ + '<script>make_collapsible(document.getElementById("heading-notes"), document.getElementById("div-notes"))();</script>\n\n') + +SOFTWARELIST_MACHINE_TABLE_HEADER = string.Template( + '<h2 id="heading-machines">Machines</h2>\n' \ + '<table id="tbl-machines">\n' \ + ' <thead>\n' \ + ' <tr>\n' \ + ' <th>Short name</th>\n' \ + ' <th>Description</th>\n' \ + ' <th>Year</th>\n' \ + ' <th>Manufacturer</th>\n' \ + ' <th>Status</th>\n' \ + ' </tr>\n' \ + ' </thead>\n' \ + ' <tbody>\n') + +SOFTWARELIST_MACHINE_TABLE_ROW = string.Template( + ' <tr>\n' \ + ' <td><a href="${machinehref}">${shortname}</a></td>\n' \ + ' <td><a href="${machinehref}">${description}</a></td>\n' \ + ' <td>${year}</td>\n' \ + ' <td>${manufacturer}</td>\n' \ + ' <td>${status}</td>\n' \ + ' </tr>\n') + +SOFTWARELIST_SOFTWARE_TABLE_HEADER = string.Template( + '<h2 id="heading-software">Software</h2>\n' \ + '<table id="tbl-software">\n' \ + ' <thead>\n' \ + ' <tr>\n' \ + ' <th>Short name</th>\n' \ + ' <th>Description</th>\n' \ + ' <th>Year</th>\n' \ + ' <th>Publisher</th>\n' \ + ' <th>Supported</th>\n' \ + ' <th class="numeric">Parts</th>\n' \ + ' <th class="numeric">Bad dumps</th>\n' \ + ' <th>Parent</th>\n' \ + ' </tr>\n' \ + ' </thead>\n' \ + ' <tbody>\n') + +SOFTWARELIST_SOFTWARE_ROW = string.Template( + ' <tr>\n' \ + ' <td><a href="${softwarehref}">${shortname}</a></td>\n' \ + ' <td><a href="${softwarehref}">${description}</a></td>\n' \ + ' <td>${year}</td>\n' \ + ' <td>${publisher}</td>\n' \ + ' <td>${supported}</td>\n' \ + ' <td style="text-align: right">${parts}</td>\n' \ + ' <td style="text-align: right">${baddumps}</td>\n' \ + ' <td>${parent}</td>\n' \ + ' </tr>\n') + + +SOFTWARELIST_LIST_PROLOGUE = string.Template( + '<!DOCTYPE html>\n' \ + '<html>\n' \ + '<head>\n' \ + ' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n' \ + ' <meta http-equiv="Content-Style-Type" content="text/css">\n' \ + ' <meta http-equiv="Content-Script-Type" content="text/javascript">\n' \ + ' <link rel="stylesheet" type="text/css" href="${assets}/style.css">\n' \ + ' <script type="text/javascript">var assetsurl="${assets}"</script>\n' \ + ' <script type="text/javascript" src="${assets}/common.js"></script>\n' \ + ' <title>${title}</title>\n' \ + '</head>\n' \ + '<body>\n' \ + '<h1>${heading}</h1>\n' \ + '<table id="tbl-softwarelists">\n' \ + ' <thead>\n' \ + ' <tr>\n' \ + ' <th>Short name</th>\n' \ + ' <th>Description</th>\n' \ + ' <th class="numeric">Total</th>\n' \ + ' <th class="numeric">Supported</th>\n' \ + ' <th class="numeric">Partially supported</th>\n' \ + ' <th class="numeric">Unsupported</th>\n' \ + ' </tr>\n' \ + ' </thead>\n' \ + ' <tbody>\n') + +SOFTWARELIST_LIST_ROW = string.Template( + ' <tr>\n' \ + ' <td><a href="${href}">${shortname}</a></td>\n' \ + ' <td><a href="${href}">${description}</a></td>\n' \ + ' <td style="text-align: right">${total}</td>\n' \ + ' <td style="text-align: right">${supported}</td>\n' \ + ' <td style="text-align: right">${partiallysupported}</td>\n' \ + ' <td style="text-align: right">${unsupported}</td>\n' \ + ' </tr>\n') + + +ROMIDENT_PAGE = string.Template( + '<!DOCTYPE html>\n' \ + '<html>\n' \ + '<head>\n' \ + ' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n' \ + ' <meta http-equiv="Content-Style-Type" content="text/css">\n' \ + ' <meta http-equiv="Content-Script-Type" content="text/javascript">\n' \ + ' <link rel="stylesheet" type="text/css" href="${assets}/style.css">\n' \ + ' <script type="text/javascript">\n' \ + ' var appurl="${app}"\n' \ + ' var assetsurl="${assets}"\n' \ + ' </script>\n' \ + ' <script type="text/javascript" src="${assets}/common.js"></script>\n' \ + ' <script type="text/javascript" src="${assets}/digest.js"></script>\n' \ + ' <script type="text/javascript" src="${assets}/romident.js"></script>\n' \ + ' <title>Identify ROM/Disk Dumps</title>\n' \ + '</head>\n' \ + '<body>\n' \ + '<h1>Identify ROM/Disk Dumps</h1>\n' \ + '<p>No files are uploaded. Files are examined locally and checksums/digests are sent to the server. File checksums and digests may be logged on the server.</p>\n' \ + '<div id="div-dropzone" class="dropzone" ondragover="div_dropzone_dragover(event)" ondrop="div_dropzone_drop(event)">\n' \ + '<p><button type="button" onclick="document.getElementById("input-dumps").click()">Select ROM/disk dumps</button></p>\n' \ + '<p>Drag and drop ROM/disk dump files here to identify them.</p>\n' \ + '</div>\n' \ + '<input id="input-dumps" type="file" multiple onchange="add_dump_files(this.files)" style="display: none">\n' \ + '<div id="div-progress"></div>\n' \ + '<div id="div-issues"></div>\n' \ + '<div id="div-unmatched"></div>\n' \ + '<div id="div-machines"></div>\n' \ + '<div id="div-software"></div>\n' \ + '</body>\n' \ + '</html>\n') diff --git a/scripts/minimaws/lib/lxparse.py b/scripts/minimaws/lib/lxparse.py index dfd8cbae45e..11c9a15f451 100644 --- a/scripts/minimaws/lib/lxparse.py +++ b/scripts/minimaws/lib/lxparse.py @@ -1,10 +1,12 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb from . import dbaccess +import os +import os.path import subprocess import xml.sax import xml.sax.saxutils @@ -12,7 +14,7 @@ import xml.sax.saxutils class ElementHandlerBase(object): def __init__(self, parent, **kwargs): - super(ElementHandlerBase, self).__init__(**kwargs) + super().__init__(**kwargs) self.dbconn = parent.dbconn if parent is not None else None self.locator = parent.locator if parent is not None else None self.depth = 0 @@ -103,7 +105,7 @@ class ElementHandler(ElementHandlerBase): class TextAccumulator(ElementHandler): def __init__(self, parent, **kwargs): - super(TextAccumulator, self).__init__(parent=parent, **kwargs) + super().__init__(parent=parent, **kwargs) self.text = '' def mainCharacters(self, content): @@ -112,7 +114,7 @@ class TextAccumulator(ElementHandler): class DipSwitchHandler(ElementHandler): def __init__(self, parent, **kwargs): - super(DipSwitchHandler, self).__init__(parent=parent, **kwargs) + super().__init__(parent=parent, **kwargs) self.dbcurs = parent.dbcurs self.machine = parent.id @@ -136,7 +138,7 @@ class DipSwitchHandler(ElementHandler): class SlotHandler(ElementHandler): def __init__(self, parent, **kwargs): - super(SlotHandler, self).__init__(parent=parent, **kwargs) + super().__init__(parent=parent, **kwargs) self.dbcurs = parent.dbcurs self.machine = parent.id @@ -153,7 +155,7 @@ class SlotHandler(ElementHandler): class RamOptionHandler(TextAccumulator): def __init__(self, parent, **kwargs): - super(RamOptionHandler, self).__init__(parent=parent, **kwargs) + super().__init__(parent=parent, **kwargs) self.dbcurs = parent.dbcurs self.machine = parent.id @@ -179,7 +181,7 @@ class MachineHandler(ElementHandler): 'ramoption': RamOptionHandler } def __init__(self, parent, **kwargs): - super(MachineHandler, self).__init__(parent=parent, **kwargs) + super().__init__(parent=parent, **kwargs) self.dbcurs = self.dbconn.cursor() def startMainElement(self, name, attrs): @@ -191,6 +193,9 @@ class MachineHandler(ElementHandler): self.romof = attrs.get('romof') self.dbcurs.add_sourcefile(self.sourcefile) + def endMainElement(self, name): + self.dbcurs.close() + def startChildElement(self, name, attrs): if name in self.CHILD_HANDLERS: self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) @@ -206,6 +211,24 @@ class MachineHandler(ElementHandler): status = 0 if 'status' not in attrs else 2 if attrs['status'] == 'unemulated' else 1 overall = status if 'overall' not in attrs else 2 if attrs['overall'] == 'unemulated' else 1 self.dbcurs.add_feature(self.id, attrs['type'], status, overall) + elif name == 'softwarelist': + self.dbcurs.add_machinesoftwarelist(self.id, attrs['name'], attrs['tag'], attrs['status']) + elif name == 'rom': + crc = attrs.get('crc') + sha1 = attrs.get('sha1') + if (crc is not None) and (sha1 is not None): + crc = int(crc, 16) + sha1 = sha1.lower() + self.dbcurs.add_rom(crc, sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_romdump(self.id, attrs['name'], crc, sha1, status != 'good') + elif name == 'disk': + sha1 = attrs.get('sha1') + if sha1 is not None: + sha1 = sha1.lower() + self.dbcurs.add_disk(sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_diskdump(self.id, attrs['name'], sha1, status != 'good') self.setChildHandler(name, attrs, self.IGNORE) def endChildHandler(self, name, handler): @@ -222,13 +245,10 @@ class MachineHandler(ElementHandler): self.manufacturer = handler.text self.dbcurs.add_system(self.id, self.year, self.manufacturer) - def endMainElement(self, name): - self.dbcurs.close() - class ListXmlHandler(ElementHandler): def __init__(self, dbconn, **kwargs): - super(ListXmlHandler, self).__init__(parent=None, **kwargs) + super().__init__(parent=None, **kwargs) self.dbconn = dbconn def startDocument(self): @@ -243,12 +263,10 @@ class ListXmlHandler(ElementHandler): msg=('Expected "mame" element but found "%s"' % (name, )), exception=None, locator=self.locator) - self.dbconn.prepare_for_load() self.machines = 0 def endMainElement(self, name): - # TODO: build index by first letter or whatever - self.dbconn.finalise_load() + self.dbconn.commit() def startChildElement(self, name, attrs): if name != 'machine': @@ -270,11 +288,184 @@ class ListXmlHandler(ElementHandler): pass +class DataAreaHandler(ElementHandler): + def __init__(self, parent, **kwargs): + super().__init__(parent=parent, **kwargs) + self.dbcurs = parent.dbcurs + self.part = parent.id + + def startChildElement(self, name, attrs): + if name == 'rom': + crc = attrs.get('crc') + sha1 = attrs.get('sha1') + if ('name' in attrs) and (crc is not None) and (sha1 is not None): + crc = int(crc, 16) + sha1 = sha1.lower() + self.dbcurs.add_rom(crc, sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_softwareromdump(self.part, attrs['name'], crc, sha1, status != 'good') + self.setChildHandler(name, attrs, self.IGNORE) + + +class DiskAreaHandler(ElementHandler): + def __init__(self, parent, **kwargs): + super().__init__(parent=parent, **kwargs) + self.dbcurs = parent.dbcurs + self.part = parent.id + + def startChildElement(self, name, attrs): + if name == 'disk': + sha1 = attrs.get('sha1') + if sha1 is not None: + sha1 = sha1.lower() + self.dbcurs.add_disk(sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_softwarediskdump(self.part, attrs['name'], sha1, status != 'good') + self.setChildHandler(name, attrs, self.IGNORE) + + +class SoftwarePartHandler(ElementHandler): + CHILD_HANDLERS = { + 'dataarea': DataAreaHandler, + 'diskarea': DiskAreaHandler } + + def __init__(self, parent, **kwargs): + super().__init__(parent=parent, **kwargs) + self.dbcurs = parent.dbcurs + self.software = parent.id + + def startMainElement(self, name, attrs): + self.id = self.dbcurs.add_softwarepart(self.software, attrs['name'], attrs['interface']) + + def startChildElement(self, name, attrs): + if name in self.CHILD_HANDLERS: + self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) + else: + if name == 'feature': + self.dbcurs.add_softwarepartfeaturetype(attrs['name']) + self.dbcurs.add_softwarepartfeature(self.id, attrs['name'], attrs['value']) + self.setChildHandler(name, attrs, self.IGNORE) + + +class SoftwareHandler(ElementHandler): + CHILD_HANDLERS = { + 'description': TextAccumulator, + 'year': TextAccumulator, + 'publisher': TextAccumulator, + 'notes': TextAccumulator, + 'part': SoftwarePartHandler } + + def __init__(self, parent, **kwargs): + super().__init__(parent=parent, **kwargs) + self.dbcurs = self.dbconn.cursor() + self.softwarelist = parent.id + + def startMainElement(self, name, attrs): + self.shortname = attrs['name'] + self.cloneof = attrs.get('cloneof') + self.supported = 0 if (attrs.get('supported', 'yes') == 'yes') else 1 if (attrs.get('supported', 'yes') == 'partial') else 2 + + def endMainElement(self, name): + self.dbcurs.close() + + def startChildElement(self, name, attrs): + if name in self.CHILD_HANDLERS: + self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) + else: + if name == 'info': + self.dbcurs.add_softwareinfotype(attrs['name']) + self.dbcurs.add_softwareinfo(self.id, attrs['name'], attrs['value']) + elif name == 'sharedfeat': + self.dbcurs.add_softwaresharedfeattype(attrs['name']) + self.dbcurs.add_softwaresharedfeat(self.id, attrs['name'], attrs['value']) + self.setChildHandler(name, attrs, self.IGNORE) + + def endChildHandler(self, name, handler): + if name == 'description': + self.description = handler.text + elif name == 'year': + self.year = handler.text + elif name == 'publisher': + self.publisher = handler.text + self.id = self.dbcurs.add_software(self.softwarelist, self.shortname, self.supported, self.description, self.year, self.publisher) + if self.cloneof is not None: + self.dbcurs.add_softwarecloneof(self.id, self.cloneof) + elif name == 'notes': + self.dbcurs.add_softwarenotes(self.id, handler.text) + + +class SoftwareListHandler(ElementHandler): + CHILD_HANDLERS = { + 'notes': TextAccumulator, + 'software': SoftwareHandler } + + def __init__(self, dbconn, **kwargs): + super().__init__(parent=None, **kwargs) + self.dbconn = dbconn + + def startDocument(self): + pass + + def endDocument(self): + pass + + def startMainElement(self, name, attrs): + if name != 'softwarelist': + raise xml.sax.SAXParseException( + msg=('Expected "softwarelist" element but found "%s"' % (name, )), + exception=None, + locator=self.locator) + self.shortname = attrs['name'] + self.description = attrs['description'] + self.entries = 0 + dbcurs = self.dbconn.cursor() + self.id = dbcurs.add_softwarelist(self.shortname, self.description) + dbcurs.close() + + def endMainElement(self, name): + self.dbconn.commit() + + def startChildElement(self, name, attrs): + if name in self.CHILD_HANDLERS: + self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) + else: + raise xml.sax.SAXParseException( + msg=('Found unexpected element "%s"' % (name, )), + exception=None, + locator=self.locator) + + def endChildHandler(self, name, handler): + if name == 'notes': + dbcurs = self.dbconn.cursor() + dbcurs.add_softwarelistnotes(self.id, handler.text) + dbcurs.close() + elif name == 'software': + if self.entries >= 1023: + self.dbconn.commit() + self.entries = 0 + else: + self.entries += 1 + + def processingInstruction(self, target, data): + pass + + def load_info(options): + dbconn = dbaccess.UpdateConnection(options.database) + dbconn.prepare_for_load() parser = xml.sax.make_parser() - parser.setContentHandler(ListXmlHandler(dbaccess.UpdateConnection(options.database))) + + parser.setContentHandler(SoftwareListHandler(dbconn)) + for path in options.softwarepath: + files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.xml')] + for filename in files: + parser.parse(filename) + + parser.setContentHandler(ListXmlHandler(dbconn)) if options.executable is not None: task = subprocess.Popen([options.executable, '-listxml'], stdout=subprocess.PIPE) parser.parse(task.stdout) else: parser.parse(options.file) + + dbconn.finalise_load() diff --git a/scripts/minimaws/lib/wsgiserve.py b/scripts/minimaws/lib/wsgiserve.py index b30828e01e0..56197407516 100644 --- a/scripts/minimaws/lib/wsgiserve.py +++ b/scripts/minimaws/lib/wsgiserve.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb @@ -6,20 +6,22 @@ from . import dbaccess from . import htmltmpl -import cgi +import html import inspect import json import mimetypes import os.path import re import sys -import wsgiref.simple_server +import urllib +import urllib.parse import wsgiref.util -if sys.version_info >= (3, ): - import urllib.parse as urlparse -else: - import urlparse +htmlescape = html.escape +shiftpath = wsgiref.util.shift_path_info +urljoin = urllib.parse.urljoin +urlparsequery = urllib.parse.parse_qs +urlquote = urllib.parse.quote class HandlerBase(object): @@ -37,7 +39,7 @@ class HandlerBase(object): 505: 'HTTP Version Not Supported' } def __init__(self, app, application_uri, environ, start_response, **kwargs): - super(HandlerBase, self).__init__(**kwargs) + super().__init__(**kwargs) self.app = app self.js_escape = app.js_escape self.application_uri = application_uri @@ -45,12 +47,12 @@ class HandlerBase(object): self.start_response = start_response def error_page(self, code): - yield htmltmpl.ERROR_PAGE.substitute(code=cgi.escape('%d' % code), message=cgi.escape(self.STATUS_MESSAGE[code])).encode('utf-8') + yield htmltmpl.ERROR_PAGE.substitute(code=htmlescape('%d' % (code, )), message=htmlescape(self.STATUS_MESSAGE[code])).encode('utf-8') class ErrorPageHandler(HandlerBase): def __init__(self, code, app, application_uri, environ, start_response, **kwargs): - super(ErrorPageHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) self.code = code self.start_response('%d %s' % (self.code, self.STATUS_MESSAGE[code]), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) @@ -59,10 +61,12 @@ class ErrorPageHandler(HandlerBase): class AssetHandler(HandlerBase): + EXTENSIONMAP = { '.js': 'application/javascript', '.svg': 'image/svg+xml' } + def __init__(self, directory, app, application_uri, environ, start_response, **kwargs): - super(AssetHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) self.directory = directory - self.asset = wsgiref.util.shift_path_info(environ) + self.asset = shiftpath(environ) def __iter__(self): if not self.asset: @@ -82,8 +86,11 @@ class AssetHandler(HandlerBase): else: try: f = open(path, 'rb') - type, encoding = mimetypes.guess_type(path) - self.start_response('200 OK', [('Content-type', type or 'application/octet-stream'), ('Cache-Control', 'public, max-age=3600')]) + base, extension = os.path.splitext(path) + mimetype = self.EXTENSIONMAP.get(extension) + if mimetype is None: + mimetype, encoding = mimetypes.guess_type(path) + self.start_response('200 OK', [('Content-type', mimetype or 'application/octet-stream'), ('Cache-Control', 'public, max-age=3600')]) return wsgiref.util.FileWrapper(f) except: self.start_response('500 %s' % (self.STATUS_MESSAGE[500], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) @@ -92,20 +99,100 @@ class AssetHandler(HandlerBase): class QueryPageHandler(HandlerBase): def __init__(self, app, application_uri, environ, start_response, **kwargs): - super(QueryPageHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) self.dbcurs = app.dbconn.cursor() def machine_href(self, shortname): - return cgi.escape(urlparse.urljoin(self.application_uri, 'machine/%s' % (shortname, )), True) + return htmlescape(urljoin(self.application_uri, 'machine/%s' % (urlquote(shortname), )), True) def sourcefile_href(self, sourcefile): - return cgi.escape(urlparse.urljoin(self.application_uri, 'sourcefile/%s' % (sourcefile, )), True) + return htmlescape(urljoin(self.application_uri, 'sourcefile/%s' % (urlquote(sourcefile), )), True) + + def softwarelist_href(self, softwarelist): + return htmlescape(urljoin(self.application_uri, 'softwarelist/%s' % (urlquote(softwarelist), )), True) + + def software_href(self, softwarelist, software): + return htmlescape(urljoin(self.application_uri, 'softwarelist/%s/%s' % (urlquote(softwarelist), urlquote(software))), True) + + def bios_data(self, machine): + result = { } + for name, description, isdefault in self.dbcurs.get_biossets(machine): + result[name] = { 'description': description, 'isdefault': True if isdefault else False } + return result + + def flags_data(self, machine): + result = { 'features': { } } + for feature, status, overall in self.dbcurs.get_feature_flags(machine): + detail = { } + if status == 1: + detail['status'] = 'imperfect' + elif status > 1: + detail['status'] = 'unemulated' + if overall == 1: + detail['overall'] = 'imperfect' + elif overall > 1: + detail['overall'] = 'unemulated' + result['features'][feature] = detail + return result + + def slot_data(self, machine): + result = { 'defaults': { }, 'slots': { } } + + # get slot options + prev = None + for slot, option, shortname, description in self.dbcurs.get_slot_options(machine): + if slot != prev: + if slot in result['slots']: + options = result['slots'][slot] + else: + options = { } + result['slots'][slot] = options + prev = slot + options[option] = { 'device': shortname, 'description': description } + + # if there are any slots, get defaults + if result['slots']: + for slot, default in self.dbcurs.get_slot_defaults(machine): + result['defaults'][slot] = default + + # remove slots that come from default cards in other slots + for slot in tuple(result['slots'].keys()): + slot += ':' + for candidate in tuple(result['slots'].keys()): + if candidate.startswith(slot): + del result['slots'][candidate] + + return result + + def softwarelist_data(self, machine): + result = { } + + # get software lists referenced by machine + for softwarelist in self.dbcurs.get_machine_softwarelists(machine): + result[softwarelist['tag']] = { + 'status': softwarelist['status'], + 'shortname': softwarelist['shortname'], + 'description': softwarelist['description'], + 'total': softwarelist['total'], + 'supported': softwarelist['supported'], + 'partiallysupported': softwarelist['partiallysupported'], + 'unsupported': softwarelist['unsupported'] } + + # remove software lists that come from default cards in slots + if result: + for slot, default in self.dbcurs.get_slot_defaults(machine): + slot += ':' + for candidate in tuple(result.keys()): + if candidate.startswith(slot): + del result[candidate] + + return result class MachineRpcHandlerBase(QueryPageHandler): def __init__(self, app, application_uri, environ, start_response, **kwargs): - super(MachineRpcHandlerBase, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) - self.shortname = wsgiref.util.shift_path_info(environ) + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + self.shortname = shiftpath(environ) def __iter__(self): if not self.shortname: @@ -129,8 +216,8 @@ class MachineRpcHandlerBase(QueryPageHandler): class MachineHandler(QueryPageHandler): def __init__(self, app, application_uri, environ, start_response, **kwargs): - super(MachineHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) - self.shortname = wsgiref.util.shift_path_info(environ) + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + self.shortname = shiftpath(environ) def __iter__(self): if not self.shortname: @@ -141,7 +228,7 @@ class MachineHandler(QueryPageHandler): self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) return self.error_page(404) else: - machine_info = self.dbcurs.get_machine_info(self.shortname).fetchone() + machine_info = self.dbcurs.get_machine_details(self.shortname).fetchone() if not machine_info: self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) return self.error_page(404) @@ -156,39 +243,39 @@ class MachineHandler(QueryPageHandler): id = machine_info['id'] description = machine_info['description'] yield htmltmpl.MACHINE_PROLOGUE.substitute( - app=self.js_escape(cgi.escape(self.application_uri, True)), - assets=self.js_escape(cgi.escape(urlparse.urljoin(self.application_uri, 'static'), True)), + app=self.js_escape(htmlescape(self.application_uri, True)), + assets=self.js_escape(htmlescape(urljoin(self.application_uri, 'static'), True)), sourcehref=self.sourcefile_href(machine_info['sourcefile']), - description=cgi.escape(description), - shortname=cgi.escape(self.shortname), - isdevice=cgi.escape('Yes' if machine_info['isdevice'] else 'No'), - runnable=cgi.escape('Yes' if machine_info['runnable'] else 'No'), - sourcefile=cgi.escape(machine_info['sourcefile'])).encode('utf-8') + description=htmlescape(description), + shortname=htmlescape(self.shortname), + isdevice=htmlescape('Yes' if machine_info['isdevice'] else 'No'), + runnable=htmlescape('Yes' if machine_info['runnable'] else 'No'), + sourcefile=htmlescape(machine_info['sourcefile'])).encode('utf-8') if machine_info['year'] is not None: yield ( ' <tr><th>Year:</th><td>%s</td></tr>\n' \ ' <tr><th>Manufacturer:</th><td>%s</td></tr>\n' % - (cgi.escape(machine_info['year']), cgi.escape(machine_info['Manufacturer']))).encode('utf-8') + (htmlescape(machine_info['year']), htmlescape(machine_info['Manufacturer']))).encode('utf-8') if machine_info['cloneof'] is not None: parent = self.dbcurs.listfull(machine_info['cloneof']).fetchone() if parent: yield ( - ' <tr><th>Parent Machine:</th><td><a href="%s">%s (%s)</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['cloneof']), True), cgi.escape(parent[1]), cgi.escape(machine_info['cloneof']))).encode('utf-8') + ' <tr><th>Parent machine:</th><td><a href="%s">%s (%s)</a></td></tr>\n' % + (self.machine_href(machine_info['cloneof']), htmlescape(parent[1]), htmlescape(machine_info['cloneof']))).encode('utf-8') else: yield ( - ' <tr><th>Parent Machine:</th><td><a href="%s">%s</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['cloneof']), True), cgi.escape(machine_info['cloneof']))).encode('utf-8') + ' <tr><th>Parent machine:</th><td><a href="%s">%s</a></td></tr>\n' % + (self.machine_href(machine_info['cloneof']), htmlescape(machine_info['cloneof']))).encode('utf-8') if (machine_info['romof'] is not None) and (machine_info['romof'] != machine_info['cloneof']): parent = self.dbcurs.listfull(machine_info['romof']).fetchone() if parent: yield ( ' <tr><th>Parent ROM set:</th><td><a href="%s">%s (%s)</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['romof']), True), cgi.escape(parent[1]), cgi.escape(machine_info['romof']))).encode('utf-8') + (self.machine_href(machine_info['romof']), htmlescape(parent[1]), htmlescape(machine_info['romof']))).encode('utf-8') else: yield ( - ' <tr><th>Parent Machine:</th><td><a href="%s">%s</a></td></tr>\n' % - (cgi.escape('%smachine/%s' % (self.application_uri, machine_info['romof']), True), cgi.escape(machine_info['romof']))).encode('utf-8') + ' <tr><th>Parent ROM set:</th><td><a href="%s">%s</a></td></tr>\n' % + (self.machine_href(machine_info['romof']), htmlescape(machine_info['romof']))).encode('utf-8') unemulated = [] imperfect = [] for feature, status, overall in self.dbcurs.get_feature_flags(id): @@ -207,6 +294,38 @@ class MachineHandler(QueryPageHandler): tuple(imperfect)).encode('utf-8'); yield '</table>\n'.encode('utf-8') + # make a table of clones + first = True + for clone, clonedescription, cloneyear, clonemanufacturer in self.dbcurs.get_clones(self.shortname): + if first: + yield htmltmpl.MACHINE_CLONES_PROLOGUE.substitute().encode('utf-8') + first = False + yield htmltmpl.MACHINE_CLONES_ROW.substitute( + href=self.machine_href(clone), + shortname=htmlescape(clone), + description=htmlescape(clonedescription), + year=htmlescape(cloneyear or ''), + manufacturer=htmlescape(clonemanufacturer or '')).encode('utf-8') + if not first: + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-clones').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-clones"), document.getElementById("tbl-clones"));</script>\n'.encode('utf-8') + + # make a table of software lists + yield htmltmpl.MACHINE_SOFTWARELISTS_TABLE_PROLOGUE.substitute().encode('utf-8') + for softwarelist in self.dbcurs.get_machine_softwarelists(id): + total = softwarelist['total'] + yield htmltmpl.MACHINE_SOFTWARELISTS_TABLE_ROW.substitute( + rowid=htmlescape(softwarelist['tag'].replace(':', '-'), True), + href=self.softwarelist_href(softwarelist['shortname']), + shortname=htmlescape(softwarelist['shortname']), + description=htmlescape(softwarelist['description']), + status=htmlescape(softwarelist['status']), + total=htmlescape('%d' % (total, )), + supported=htmlescape('%.1f%%' % (softwarelist['supported'] * 100.0 / (total or 1), )), + partiallysupported=htmlescape('%.1f%%' % (softwarelist['partiallysupported'] * 100.0 / (total or 1), )), + unsupported=htmlescape('%.1f%%' % (softwarelist['unsupported'] * 100.0 / (total or 1), ))).encode('utf-8') + yield htmltmpl.MACHINE_SOFTWARELISTS_TABLE_EPILOGUE.substitute().encode('utf-8') + # allow system BIOS selection haveoptions = False for name, desc, isdef in self.dbcurs.get_biossets(id): @@ -215,8 +334,8 @@ class MachineHandler(QueryPageHandler): yield htmltmpl.MACHINE_OPTIONS_HEADING.substitute().encode('utf-8') yield htmltmpl.MACHINE_BIOS_PROLOGUE.substitute().encode('utf-8') yield htmltmpl.MACHINE_BIOS_OPTION.substitute( - name=cgi.escape(name, True), - description=cgi.escape(desc), + name=htmlescape(name, True), + description=htmlescape(desc), isdefault=('yes' if isdef else 'no')).encode('utf-8') if haveoptions: yield '</select>\n<script>set_default_system_bios();</script>\n'.encode('utf-8') @@ -231,26 +350,51 @@ class MachineHandler(QueryPageHandler): yield htmltmpl.MACHINE_RAM_PROLOGUE.substitute().encode('utf-8') first = False yield htmltmpl.MACHINE_RAM_OPTION.substitute( - name=cgi.escape(name, True), - size=cgi.escape('{:,}'.format(size)), + name=htmlescape(name, True), + size=htmlescape('{:,}'.format(size)), isdefault=('yes' if isdef else 'no')).encode('utf-8') if not first: - yield '</select>\n<script>set_default_ram_option();</script>\n'.encode('utf-8') + yield ' </select>\n <script>set_default_ram_option();</script>\n'.encode('utf-8') # placeholder for machine slots - populated by client-side JavaScript if self.dbcurs.count_slots(id): if not haveoptions: haveoptions = True yield htmltmpl.MACHINE_OPTIONS_HEADING.substitute().encode('utf-8') - yield htmltmpl.MACHINE_SLOTS_PLACEHOLDER.substitute( - machine=self.js_escape(self.shortname)).encode('utf=8') + yield htmltmpl.MACHINE_SLOTS_PLACEHOLDER_PROLOGUE.substitute().encode('utf=8') + pending = set((self.shortname, )) + added = set((self.shortname, )) + haveextra = set() + while pending: + requested = pending.pop() + slots = self.slot_data(self.dbcurs.get_machine_id(requested)) + yield (' slot_info[%s] = %s;\n' % (self.sanitised_json(requested), self.sanitised_json(slots))).encode('utf-8') + for slotname, slot in slots['slots'].items(): + for choice, card in slot.items(): + carddev = card['device'] + if carddev not in added: + pending.add(carddev) + added.add(carddev) + if (carddev not in haveextra) and (slots['defaults'].get(slotname) == choice): + haveextra.add(carddev) + cardid = self.dbcurs.get_machine_id(carddev) + carddev = self.sanitised_json(carddev) + yield ( + ' bios_sets[%s] = %s;\n machine_flags[%s] = %s;\n softwarelist_info[%s] = %s;\n' % + (carddev, self.sanitised_json(self.bios_data(cardid)), carddev, self.sanitised_json(self.flags_data(cardid)), carddev, self.sanitised_json(self.softwarelist_data(cardid)))).encode('utf-8') + yield htmltmpl.MACHINE_SLOTS_PLACEHOLDER_EPILOGUE.substitute( + machine=self.sanitised_json(self.shortname)).encode('utf=8') + + # add disclosure triangle for options if present + if haveoptions: + yield htmltmpl.MACHINE_OPTIONS_EPILOGUE.substitute().encode('utf=8') # list devices referenced by this system/device first = True for name, desc, src in self.dbcurs.get_devices_referenced(id): if first: yield \ - '<h2>Devices Referenced</h2>\n' \ + '<h2 id="heading-dev-refs">Devices Referenced</h2>\n' \ '<table id="tbl-dev-refs">\n' \ ' <thead>\n' \ ' <tr><th>Short name</th><th>Description</th><th>Source file</th></tr>\n' \ @@ -260,13 +404,14 @@ class MachineHandler(QueryPageHandler): yield self.machine_row(name, desc, src) if not first: yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-dev-refs').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-dev-refs"), document.getElementById("tbl-dev-refs"));</script>\n'.encode('utf-8') # list slots where this device is an option first = True for name, desc, slot, opt, src in self.dbcurs.get_compatible_slots(id): if (first): yield \ - '<h2>Compatible Slots</h2>\n' \ + '<h2 id="heading-comp-slots">Compatible Slots</h2>\n' \ '<table id="tbl-comp-slots">\n' \ ' <thead>\n' \ ' <tr><th>Short name</th><th>Description</th><th>Slot</th><th>Choice</th><th>Source file</th></tr>\n' \ @@ -276,20 +421,21 @@ class MachineHandler(QueryPageHandler): yield htmltmpl.COMPATIBLE_SLOT_ROW.substitute( machinehref=self.machine_href(name), sourcehref=self.sourcefile_href(src), - shortname=cgi.escape(name), - description=cgi.escape(desc), - sourcefile=cgi.escape(src), - slot=cgi.escape(slot), - slotoption=cgi.escape(opt)).encode('utf-8') + shortname=htmlescape(name), + description=htmlescape(desc), + sourcefile=htmlescape(src), + slot=htmlescape(slot), + slotoption=htmlescape(opt)).encode('utf-8') if not first: yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-comp-slots').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-comp-slots"), document.getElementById("tbl-comp-slots"));</script>\n'.encode('utf-8') # list systems/devices that reference this device first = True for name, desc, src in self.dbcurs.get_device_references(id): if first: yield \ - '<h2>Referenced By</h2>\n' \ + '<h2 id="heading-ref-by">Referenced By</h2>\n' \ '<table id="tbl-ref-by">\n' \ ' <thead>\n' \ ' <tr><th>Short name</th><th>Description</th><th>Source file</th></tr>\n' \ @@ -299,6 +445,7 @@ class MachineHandler(QueryPageHandler): yield self.machine_row(name, desc, src) if not first: yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-ref-by').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-ref-by"), document.getElementById("tbl-ref-by"));</script>\n'.encode('utf-8') yield '</html>\n'.encode('utf-8') @@ -306,14 +453,18 @@ class MachineHandler(QueryPageHandler): return (htmltmpl.MACHINE_ROW if description is not None else htmltmpl.EXCL_MACHINE_ROW).substitute( machinehref=self.machine_href(shortname), sourcehref=self.sourcefile_href(sourcefile), - shortname=cgi.escape(shortname), - description=cgi.escape(description or ''), - sourcefile=cgi.escape(sourcefile or '')).encode('utf-8') + shortname=htmlescape(shortname), + description=htmlescape(description or ''), + sourcefile=htmlescape(sourcefile or '')).encode('utf-8') + + @staticmethod + def sanitised_json(data): + return json.dumps(data).replace('<', '\\u003c').replace('>', '\\u003e') class SourceFileHandler(QueryPageHandler): def __init__(self, app, application_uri, environ, start_response, **kwargs): - super(SourceFileHandler, self).__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) def __iter__(self): self.filename = self.environ['PATH_INFO'] @@ -355,21 +506,21 @@ class SourceFileHandler(QueryPageHandler): title = heading = 'All Source Files' else: heading = self.linked_title(pattern) - title = 'Source Files: ' + cgi.escape(pattern) + title = 'Source Files: ' + htmlescape(pattern) yield htmltmpl.SOURCEFILE_LIST_PROLOGUE.substitute( - assets=cgi.escape(urlparse.urljoin(self.application_uri, 'static'), True), + assets=htmlescape(urljoin(self.application_uri, 'static'), True), title=title, heading=heading).encode('utf-8') for filename, machines in self.dbcurs.get_sourcefiles(pattern): yield htmltmpl.SOURCEFILE_LIST_ROW.substitute( sourcefile=self.linked_title(filename, True), - machines=cgi.escape('%d' % machines)).encode('utf-8') - yield ' </tbody>\n</table>\n<script>make_table_sortable(document.getElementById("tbl-sourcefiles"));</script>\n</body>\n</html>\n'.encode('utf-8') + machines=htmlescape('%d' % (machines, ))).encode('utf-8') + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-sourcefiles').encode('utf-8') def sourcefile_page(self, id): yield htmltmpl.SOURCEFILE_PROLOGUE.substitute( - assets=cgi.escape(urlparse.urljoin(self.application_uri, 'static'), True), - filename=cgi.escape(self.filename), + assets=htmlescape(urljoin(self.application_uri, 'static'), True), + filename=htmlescape(self.filename), title=self.linked_title(self.filename)).encode('utf-8') first = True @@ -393,7 +544,7 @@ class SourceFileHandler(QueryPageHandler): if first: yield '<p>No machines found.</p>\n'.encode('utf-8') else: - yield ' </tbody>\n</table>\n<script>make_table_sortable(document.getElementById("tbl-machines"));</script>\n'.encode('utf-8') + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-machines').encode('utf-8') yield '</body>\n</html>\n'.encode('utf-8') @@ -401,14 +552,14 @@ class SourceFileHandler(QueryPageHandler): parts = filename.split('/') final = parts[-1] del parts[-1] - uri = urlparse.urljoin(self.application_uri, 'sourcefile') + uri = urljoin(self.application_uri, 'sourcefile') title = '' for part in parts: - uri = urlparse.urljoin(uri + '/', part) - title += '<a href="{0}">{1}</a>/'.format(cgi.escape(uri, True), cgi.escape(part)) + uri = urljoin(uri + '/', urlquote(part)) + title += '<a href="{0}">{1}</a>/'.format(htmlescape(uri, True), htmlescape(part)) if linkfinal: - uri = urlparse.urljoin(uri + '/', final) - return title + '<a href="{0}">{1}</a>'.format(cgi.escape(uri, True), cgi.escape(final)) + uri = urljoin(uri + '/', urlquote(final)) + return title + '<a href="{0}">{1}</a>'.format(htmlescape(uri, True), htmlescape(final)) else: return title + final @@ -416,12 +567,245 @@ class SourceFileHandler(QueryPageHandler): return (htmltmpl.SOURCEFILE_ROW_PARENT if machine_info['cloneof'] is None else htmltmpl.SOURCEFILE_ROW_CLONE).substitute( machinehref=self.machine_href(machine_info['shortname']), parenthref=self.machine_href(machine_info['cloneof'] or '__invalid'), - shortname=cgi.escape(machine_info['shortname']), - description=cgi.escape(machine_info['description']), - year=cgi.escape(machine_info['year'] or ''), - manufacturer=cgi.escape(machine_info['manufacturer'] or ''), - runnable=cgi.escape('Yes' if machine_info['runnable'] else 'No'), - parent=cgi.escape(machine_info['cloneof'] or '')).encode('utf-8') + shortname=htmlescape(machine_info['shortname']), + description=htmlescape(machine_info['description']), + year=htmlescape(machine_info['year'] or ''), + manufacturer=htmlescape(machine_info['manufacturer'] or ''), + runnable=htmlescape('Yes' if machine_info['runnable'] else 'No'), + parent=htmlescape(machine_info['cloneof'] or '')).encode('utf-8') + + +class SoftwareListHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + self.shortname = shiftpath(environ) + self.software = shiftpath(environ) + + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.software and ('*' not in self.software) and ('?' not in self.software): + software_info = self.dbcurs.get_software_details(self.shortname, self.software).fetchone() + if not software_info: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + self.start_response('200 OK', [('Content-type', 'text/html; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.software_page(software_info) + elif self.software or (self.shortname and ('*' not in self.shortname) and ('?' not in self.shortname)): + softwarelist_info = self.dbcurs.get_softwarelist_details(self.shortname, self.software or None).fetchone() + if not softwarelist_info: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + self.start_response('200 OK', [('Content-type', 'text/html; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.softwarelist_page(softwarelist_info, self.software or None) + else: + if self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + self.start_response('200 OK', [('Content-type', 'text/html; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.softwarelist_listing_page(self.shortname or None) + + def softwarelist_listing_page(self, pattern): + if not pattern: + title = heading = 'All Software Lists' + else: + title = heading = 'Software Lists: ' + htmlescape(pattern) + yield htmltmpl.SOFTWARELIST_LIST_PROLOGUE.substitute( + assets=htmlescape(urljoin(self.application_uri, 'static'), True), + title=title, + heading=heading).encode('utf-8') + for shortname, description, total, supported, partiallysupported, unsupported in self.dbcurs.get_softwarelists(pattern): + yield htmltmpl.SOFTWARELIST_LIST_ROW.substitute( + href=self.softwarelist_href(shortname), + shortname=htmlescape(shortname), + description=htmlescape(description), + total=htmlescape('%d' % (total, )), + supported=htmlescape('%.1f%%' % (supported * 100.0 / (total or 1), )), + partiallysupported=htmlescape('%.1f%%' % (partiallysupported * 100.0 / (total or 1), )), + unsupported=htmlescape('%.1f%%' % (unsupported * 100.0 / (total or 1), ))).encode('utf-8') + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-softwarelists').encode('utf-8') + + def softwarelist_page(self, softwarelist_info, pattern): + if not pattern: + title = 'Software List: %s (%s)' % (htmlescape(softwarelist_info['description']), htmlescape(softwarelist_info['shortname'])) + heading = htmlescape(softwarelist_info['description']) + else: + title = 'Software List: %s (%s): %s' % (htmlescape(softwarelist_info['description']), htmlescape(softwarelist_info['shortname']), htmlescape(pattern)) + heading = '<a href="%s">%s</a>: %s' % (self.softwarelist_href(softwarelist_info['shortname']), htmlescape(softwarelist_info['description']), htmlescape(pattern)) + yield htmltmpl.SOFTWARELIST_PROLOGUE.substitute( + assets=htmlescape(urljoin(self.application_uri, 'static'), True), + title=title, + heading=heading, + shortname=htmlescape(softwarelist_info['shortname']), + total=htmlescape('%d' % (softwarelist_info['total'], )), + supported=htmlescape('%d' % (softwarelist_info['supported'], )), + supportedpc=htmlescape('%.1f' % (softwarelist_info['supported'] * 100.0 / (softwarelist_info['total'] or 1), )), + partiallysupported=htmlescape('%d' % (softwarelist_info['partiallysupported'], )), + partiallysupportedpc=htmlescape('%.1f' % (softwarelist_info['partiallysupported'] * 100.0 / (softwarelist_info['total'] or 1), )), + unsupported=htmlescape('%d' % (softwarelist_info['unsupported'], )), + unsupportedpc=htmlescape('%.1f' % (softwarelist_info['unsupported'] * 100.0 / (softwarelist_info['total'] or 1), ))).encode('utf-8') + + if softwarelist_info['notes'] is not None: + yield htmltmpl.SOFTWARELIST_NOTES_PROLOGUE.substitute().encode('utf-8') + first = True + for line in softwarelist_info['notes'].strip().splitlines(): + if line: + yield (('<p>%s' if first else '<br />\n%s') % (htmlescape(line), )).encode('utf-8') + first = False + elif not first: + yield '</p>\n'.encode('utf-8') + first = True + if not first: + yield '</p>\n'.encode('utf-8') + yield htmltmpl.SOFTWARELIST_NOTES_EPILOGUE.substitute().encode('utf-8') + + first = True + for machine_info in self.dbcurs.get_softwarelist_machines(softwarelist_info['id']): + if first: + yield htmltmpl.SOFTWARELIST_MACHINE_TABLE_HEADER.substitute().encode('utf-8') + first = False + yield htmltmpl.SOFTWARELIST_MACHINE_TABLE_ROW.substitute( + machinehref=self.machine_href(machine_info['shortname']), + shortname=htmlescape(machine_info['shortname']), + description=htmlescape(machine_info['description']), + year=htmlescape(machine_info['year'] or ''), + manufacturer=htmlescape(machine_info['manufacturer'] or ''), + status=htmlescape(machine_info['status'])).encode('utf-8') + if not first: + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-machines').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-machines"), document.getElementById("tbl-machines"));</script>\n'.encode('utf-8') + + first = True + for software_info in self.dbcurs.get_softwarelist_software(softwarelist_info['id'], self.software or None): + if first: + yield htmltmpl.SOFTWARELIST_SOFTWARE_TABLE_HEADER.substitute().encode('utf-8') + first = False + yield self.software_row(software_info) + if first: + yield '<p>No software found.</p>\n'.encode('utf-8') + else: + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-software').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-software"), document.getElementById("tbl-software"));</script>\n'.encode('utf-8') + + yield '</body>\n</html>\n'.encode('utf-8') + + def software_page(self, software_info): + yield htmltmpl.SOFTWARE_PROLOGUE.substitute( + assets=htmlescape(urljoin(self.application_uri, 'static'), True), + title=htmlescape(software_info['description']), + heading=htmlescape(software_info['description']), + softwarelisthref=self.softwarelist_href(self.shortname), + softwarelistdescription=htmlescape(software_info['softwarelistdescription']), + softwarelist=htmlescape(self.shortname), + shortname=htmlescape(software_info['shortname']), + year=htmlescape(software_info['year']), + publisher=htmlescape(software_info['publisher'])).encode('utf-8') + if software_info['parent'] is not None: + yield (' <tr><th>Parent:</th><td><a href="%s">%s</a></td>\n' % (self.software_href(software_info['parentsoftwarelist'], software_info['parent']), htmlescape(software_info['parentdescription']))).encode('utf-8') + yield (' <tr><th>Supported:</th><td>%s</td>\n' % (self.format_supported(software_info['supported']), )).encode('utf-8') + for name, value in self.dbcurs.get_software_info(software_info['id']): + yield (' <tr><th>%s:</th><td>%s</td>\n' % (htmlescape(name), htmlescape(value))).encode('utf-8') + yield '</table>\n\n'.encode('utf-8') + + first = True + for clone_info in self.dbcurs.get_software_clones(software_info['id']): + if first: + yield htmltmpl.SOFTWARE_CLONES_PROLOGUE.substitute().encode('utf-8') + first = False + yield self.clone_row(clone_info) + if not first: + yield htmltmpl.SORTABLE_TABLE_EPILOGUE.substitute(id='tbl-clones').encode('utf-8') + yield '<script>make_collapsible(document.getElementById("heading-clones"), document.getElementById("tbl-clones"));</script>\n'.encode('utf-8') + + if software_info['notes'] is not None: + yield htmltmpl.SOFTWARE_NOTES_PROLOGUE.substitute().encode('utf-8') + first = True + for line in software_info['notes'].strip().splitlines(): + if line: + yield (('<p>%s' if first else '<br />\n%s') % (htmlescape(line), )).encode('utf-8') + first = False + elif not first: + yield '</p>\n'.encode('utf-8') + first = True + if not first: + yield '</p>\n'.encode('utf-8') + yield htmltmpl.SOFTWARE_NOTES_EPILOGUE.substitute().encode('utf-8') + + parts = self.dbcurs.get_software_parts(software_info['id']).fetchall() + first = True + for id, partname, interface, part_id in parts: + if first: + yield htmltmpl.SOFTWARE_PARTS_PROLOGUE.substitute().encode('utf-8') + first = False + yield htmltmpl.SOFTWARE_PART_PROLOGUE.substitute( + heading=htmlescape(('%s (%s)' % (part_id, partname)) if part_id is not None else partname), + shortname=htmlescape(partname), + interface=htmlescape(interface)).encode('utf-8') + for name, value in self.dbcurs.get_softwarepart_features(id): + yield (' <tr><th>%s:</th><td>%s</td>\n' % (htmlescape(name), htmlescape(value))).encode('utf-8') + yield ' </table>\n\n'.encode('utf-8') + if not first: + yield htmltmpl.SOFTWARE_PARTS_EPILOGUE.substitute().encode('utf-8') + + yield '</body>\n</html>\n'.encode('utf-8') + + def software_row(self, software_info): + parent = software_info['parent'] + return htmltmpl.SOFTWARELIST_SOFTWARE_ROW.substitute( + softwarehref=self.software_href(self.shortname, software_info['shortname']), + shortname=htmlescape(software_info['shortname']), + description=htmlescape(software_info['description']), + year=htmlescape(software_info['year']), + publisher=htmlescape(software_info['publisher']), + supported=self.format_supported(software_info['supported']), + parts=htmlescape('%d' % (software_info['parts'], )), + baddumps=htmlescape('%d' % (software_info['baddumps'], )), + parent='<a href="%s">%s</a>' % (self.software_href(software_info['parentsoftwarelist'], parent), htmlescape(parent)) if parent is not None else '').encode('utf-8') + + def clone_row(self, clone_info): + return htmltmpl.SOFTWARE_CLONES_ROW.substitute( + href=self.software_href(clone_info['softwarelist'], clone_info['shortname']), + shortname=htmlescape(clone_info['shortname']), + description=htmlescape(clone_info['description']), + year=htmlescape(clone_info['year']), + publisher=htmlescape(clone_info['publisher']), + supported=self.format_supported(clone_info['supported'])).encode('utf-8') + + @staticmethod + def format_supported(supported): + return 'Yes' if supported == 0 else 'Partial' if supported == 1 else 'No' + + +class RomIdentHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + self.dbcurs = app.dbconn.cursor() + + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + self.start_response('200 OK', [('Content-type', 'text/html; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.form_page() + + def form_page(self): + yield htmltmpl.ROMIDENT_PAGE.substitute( + app=self.js_escape(htmlescape(self.application_uri, True)), + assets=self.js_escape(htmlescape(urljoin(self.application_uri, 'static'), True))).encode('utf-8') class BiosRpcHandler(MachineRpcHandlerBase): @@ -434,58 +818,143 @@ class BiosRpcHandler(MachineRpcHandlerBase): class FlagsRpcHandler(MachineRpcHandlerBase): def data_page(self, machine): - result = { 'features': { } } - for feature, status, overall in self.dbcurs.get_feature_flags(machine): - detail = { } - if status == 1: - detail['status'] = 'imperfect' - elif status > 1: - detail['status'] = 'unemulated' - if overall == 1: - detail['overall'] = 'imperfect' - elif overall > 1: - detail['overall'] = 'unemulated' - result['features'][feature] = detail - yield json.dumps(result).encode('utf-8') + yield json.dumps(self.flags_data(machine)).encode('utf-8') class SlotsRpcHandler(MachineRpcHandlerBase): def data_page(self, machine): - result = { 'defaults': { }, 'slots': { } } + yield json.dumps(self.slot_data(machine)).encode('utf-8') + + +class SoftwareListsRpcHandler(MachineRpcHandlerBase): + def data_page(self, machine): + yield json.dumps(self.softwarelist_data(machine)).encode('utf-8') + + +class RomDumpsRpcHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + try: + args = urlparsequery(self.environ['QUERY_STRING'], keep_blank_values=True, strict_parsing=True) + crc = args.get('crc') + sha1 = args.get('sha1') + if (len(args) == 2) and (crc is not None) and (len(crc) == 1) and (sha1 is not None) and (len(sha1) == 1): + crc = int(crc[0], 16) + sha1 = sha1[0] + self.start_response('200 OK', [('Content-type', 'application/json; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.data_page(crc, sha1) + except BaseException as e: + pass + self.start_response('500 %s' % (self.STATUS_MESSAGE[500], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(500) + + def data_page(self, crc, sha1): + machines = { } + for shortname, description, label, bad in self.dbcurs.get_rom_dumps(crc, sha1): + machine = machines.get(shortname) + if machine is None: + machine = { 'description': description, 'matches': [ ] } + machines[shortname] = machine + machine['matches'].append({ 'name': label, 'bad': bool(bad) }) + + software = { } + for softwarelist, softwarelistdescription, shortname, description, part, part_id, label, bad in self.dbcurs.get_software_rom_dumps(crc, sha1): + listinfo = software.get(softwarelist) + if listinfo is None: + listinfo = { 'description': softwarelistdescription, 'software': { } } + software[softwarelist] = listinfo + softwareinfo = listinfo['software'].get(shortname) + if softwareinfo is None: + softwareinfo = { 'description': description, 'parts': { } } + listinfo['software'][shortname] = softwareinfo + partinfo = softwareinfo['parts'].get(part) + if partinfo is None: + partinfo = { 'matches': [ ] } + if part_id is not None: + partinfo['description'] = part_id + softwareinfo['parts'][part] = partinfo + partinfo['matches'].append({ 'name': label, 'bad': bool(bad) }) + + result = { 'machines': machines, 'software': software } + yield json.dumps(result).encode('utf-8') - # get defaults and slot options - for slot, default in self.dbcurs.get_slot_defaults(machine): - result['defaults'][slot] = default - prev = None - for slot, option, shortname, description in self.dbcurs.get_slot_options(machine): - if slot != prev: - if slot in result['slots']: - options = result['slots'][slot] - else: - options = { } - result['slots'][slot] = options - prev = slot - options[option] = { 'device': shortname, 'description': description } - # remove slots that come from default cards in other slots - for slot in tuple(result['slots'].keys()): - slot += ':' - for candidate in tuple(result['slots'].keys()): - if candidate.startswith(slot): - del result['slots'][candidate] +class DiskDumpsRpcHandler(QueryPageHandler): + def __init__(self, app, application_uri, environ, start_response, **kwargs): + super().__init__(app=app, application_uri=application_uri, environ=environ, start_response=start_response, **kwargs) + def __iter__(self): + if self.environ['PATH_INFO']: + self.start_response('404 %s' % (self.STATUS_MESSAGE[404], ), [('Content-type', 'text/html; charset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(404) + elif self.environ['REQUEST_METHOD'] != 'GET': + self.start_response('405 %s' % (self.STATUS_MESSAGE[405], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(405) + else: + try: + args = urlparsequery(self.environ['QUERY_STRING'], keep_blank_values=True, strict_parsing=True) + sha1 = args.get('sha1') + if (len(args) == 1) and (sha1 is not None) and (len(sha1) == 1): + sha1 = sha1[0] + self.start_response('200 OK', [('Content-type', 'application/json; chearset=utf-8'), ('Cache-Control', 'public, max-age=3600')]) + return self.data_page(sha1) + except BaseException as e: + pass + self.start_response('500 %s' % (self.STATUS_MESSAGE[500], ), [('Content-type', 'text/html; charset=utf-8'), ('Accept', 'GET, HEAD, OPTIONS'), ('Cache-Control', 'public, max-age=3600')]) + return self.error_page(500) + + def data_page(self, sha1): + machines = { } + for shortname, description, label, bad in self.dbcurs.get_disk_dumps(sha1): + machine = machines.get(shortname) + if machine is None: + machine = { 'description': description, 'matches': [ ] } + machines[shortname] = machine + machine['matches'].append({ 'name': label, 'bad': bool(bad) }) + + software = { } + for softwarelist, softwarelistdescription, shortname, description, part, part_id, label, bad in self.dbcurs.get_software_disk_dumps(sha1): + listinfo = software.get(softwarelist) + if listinfo is None: + listinfo = { 'description': softwarelistdescription, 'software': { } } + software[softwarelist] = listinfo + softwareinfo = listinfo['software'].get(shortname) + if softwareinfo is None: + softwareinfo = { 'description': description, 'parts': { } } + listinfo['software'][shortname] = softwareinfo + partinfo = softwareinfo['parts'].get(part) + if partinfo is None: + partinfo = { 'matches': [ ] } + if part_id is not None: + partinfo['description'] = part_id + softwareinfo['parts'][part] = partinfo + partinfo['matches'].append({ 'name': label, 'bad': bool(bad) }) + + result = { 'machines': machines, 'software': software } yield json.dumps(result).encode('utf-8') class MiniMawsApp(object): JS_ESCAPE = re.compile('([\"\'\\\\])') RPC_SERVICES = { - 'bios': BiosRpcHandler, - 'flags': FlagsRpcHandler, - 'slots': SlotsRpcHandler } + 'bios': BiosRpcHandler, + 'flags': FlagsRpcHandler, + 'slots': SlotsRpcHandler, + 'softwarelists': SoftwareListsRpcHandler, + 'romdumps': RomDumpsRpcHandler, + 'diskdumps': DiskDumpsRpcHandler } def __init__(self, dbfile, **kwargs): - super(MiniMawsApp, self).__init__(**kwargs) + super().__init__(**kwargs) self.dbconn = dbaccess.QueryConnection(dbfile) self.assetsdir = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), 'assets') if not mimetypes.inited: @@ -493,15 +962,21 @@ class MiniMawsApp(object): def __call__(self, environ, start_response): application_uri = wsgiref.util.application_uri(environ) - module = wsgiref.util.shift_path_info(environ) + if application_uri[-1] != '/': + application_uri += '/' + module = shiftpath(environ) if module == 'machine': return MachineHandler(self, application_uri, environ, start_response) elif module == 'sourcefile': return SourceFileHandler(self, application_uri, environ, start_response) + elif module == 'softwarelist': + return SoftwareListHandler(self, application_uri, environ, start_response) + elif module == 'romident': + return RomIdentHandler(self, application_uri, environ, start_response) elif module == 'static': return AssetHandler(self.assetsdir, self, application_uri, environ, start_response) elif module == 'rpc': - service = wsgiref.util.shift_path_info(environ) + service = shiftpath(environ) if not service: return ErrorPageHandler(403, self, application_uri, environ, start_response) elif service in self.RPC_SERVICES: @@ -515,12 +990,3 @@ class MiniMawsApp(object): def js_escape(self, str): return self.JS_ESCAPE.sub('\\\\\\1', str).replace('\0', '\\0') - - -def run_server(options): - application = MiniMawsApp(options.database) - server = wsgiref.simple_server.make_server(options.host, options.port, application) - try: - server.serve_forever() - except KeyboardInterrupt: - pass diff --git a/scripts/minimaws/minimaws.py b/scripts/minimaws/minimaws.py index 656956a0275..72e1803ed8e 100755 --- a/scripts/minimaws/minimaws.py +++ b/scripts/minimaws/minimaws.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 ## ## license:BSD-3-Clause ## copyright-holders:Vas Crabb @@ -26,11 +26,17 @@ ## $ python minimaws.py listclones "unkch*" ## $ python minimaws.py listbrothers superx ## +## The romident command does not support 7zip archives, but it's far +## faster than using MAME as it has optimised indexes, and results are +## grouped by machine rather than by file: +## +## $ python minimaws.py romident 27c64.bin dump-dir +## ## One more sophisticated query command is provided that MAME has no ## equivalent for. The listaffected command shows all runnable machines ## that reference devices defined in specified source files: ## -## $ python minimaws.py listaffected "src/devices/cpu/m6805/*" src/devices/cpu/mcs40/mcs40.cpp +## $ python minimaws.py listaffected "devices/cpu/m6805/*" devices/cpu/mcs40/mcs40.cpp ## ## This script can also run a local web server allowing you to explore ## systems, devices and source files: @@ -71,16 +77,15 @@ ## On any of these, and many other systems, you can select slot options ## and see dependent slots update. Required command-line arguments to ## produce the selected configuration are also displayed. - -import lib.auxverbs -import lib.lxparse -import lib.wsgiserve - -import argparse -import sys - +## +## Media files can be identified using a web browser interface +## +## Checksums and digests are calculated locally - no files are uploaded +## to the server. if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() parser.add_argument('--database', metavar='<dbfile>', default='minimaws.sqlite3', help='SQLite 3 info database file (defaults to minimaws.sqlite3)') subparsers = parser.add_subparsers(title='commands', dest='command', metavar='<command>') @@ -100,6 +105,9 @@ if __name__ == '__main__': subparser = subparsers.add_parser('listaffected', help='show drivers affected by source change(s)') subparser.add_argument('pattern', nargs='+', metavar='<pat>', help='source file glob pattern') + subparser = subparsers.add_parser('romident', help='identify ROM dump(s)') + subparser.add_argument('path', nargs='+', metavar='<path>', help='ROM dump file/directory path') + subparser = subparsers.add_parser('serve', help='serve over HTTP') subparser.add_argument('--port', metavar='<port>', default=8080, type=int, help='server TCP port') subparser.add_argument('--host', metavar='<host>', default='', help='server TCP hostname') @@ -108,8 +116,11 @@ if __name__ == '__main__': group = subparser.add_mutually_exclusive_group(required=True) group.add_argument('--executable', metavar='<exe>', help='emulator executable') group.add_argument('--file', metavar='<xmlfile>', help='XML machine information file') + subparser.add_argument('--softwarepath', required=True, action='append', metavar='<path>', help='Software list directory path') options = parser.parse_args() + + import lib.auxverbs if options.command == 'listfull': lib.auxverbs.do_listfull(options) elif options.command == 'listsource': @@ -120,7 +131,17 @@ if __name__ == '__main__': lib.auxverbs.do_listbrothers(options) elif options.command == 'listaffected': lib.auxverbs.do_listaffected(options) + elif options.command == 'romident': + lib.auxverbs.do_romident(options) elif options.command == 'serve': - lib.wsgiserve.run_server(options) + import wsgiref.simple_server + import lib.wsgiserve + application = lib.wsgiserve.MiniMawsApp(options.database) + server = wsgiref.simple_server.make_server(options.host, options.port, application) + try: + server.serve_forever() + except KeyboardInterrupt: + pass elif options.command == 'load': + import lib.lxparse lib.lxparse.load_info(options) diff --git a/scripts/minimaws/minimaws.wsgi b/scripts/minimaws/minimaws.wsgi new file mode 100644 index 00000000000..1ddab440be4 --- /dev/null +++ b/scripts/minimaws/minimaws.wsgi @@ -0,0 +1,16 @@ +#!/usr/bin/python +## +## license:BSD-3-Clause +## copyright-holders:Vas Crabb +## +## Simple wrapper for deploying minimaws with mod_wsgi. + +import os.path +import sys + +scriptdir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, scriptdir) + +import lib.wsgiserve + +application = lib.wsgiserve.MiniMawsApp(os.path.join(scriptdir, 'minimaws.sqlite3')) |