diff options
author | 2017-08-02 20:00:12 +1000 | |
---|---|---|
committer | 2017-08-02 20:13:13 +1000 | |
commit | c85c7c4c15fb85b98e06667b72edda2c93cf1a92 (patch) | |
tree | 7b89d1e96a7a17249fa7d5af6f698e76535f7bd8 /scripts/minimaws | |
parent | bcef3c04451c7e8884691bcb99eb129c5073e4c8 (diff) |
Load slot info in minimaws (not exposed yet), fix up some problems with
slots uncovered
(nw) It seems at some point someone didn't realise that choosing the
same option from a SLOT_INTERFACE in multiple slots creates multiple
instances of the same device type, and this got copy/pasted everywhere.
Diffstat (limited to 'scripts/minimaws')
-rw-r--r-- | scripts/minimaws/lib/dbaccess.py | 74 | ||||
-rw-r--r-- | scripts/minimaws/lib/lxparse.py | 45 | ||||
-rw-r--r-- | scripts/minimaws/lib/wsgiserve.py | 5 | ||||
-rw-r--r-- | scripts/minimaws/schema.sql | 27 |
4 files changed, 123 insertions, 28 deletions
diff --git a/scripts/minimaws/lib/dbaccess.py b/scripts/minimaws/lib/dbaccess.py index e6ff07261fe..9aaa57473b9 100644 --- a/scripts/minimaws/lib/dbaccess.py +++ b/scripts/minimaws/lib/dbaccess.py @@ -7,6 +7,14 @@ import sqlite3 class SchemaQueries(object): + CREATE_TEMPORARY_DEVICEREFERENCE = 'CREATE TEMPORARY TABLE temp_devicereference (id INTEGER PRIMARY KEY, machine INTEGER NOT NULL, device TEXT NOT NULL, UNIQUE (machine, device))' + CREATE_TEMPORARY_SLOTOPTION = 'CREATE TEMPORARY TABLE temp_slotoption (id INTEGER PRIMARY KEY, slot INTEGER NOT NULL, device TEXT NOT NULL, name TEXT NOT NULL)' + CREATE_TEMPORARY_SLOTDEFAULT = 'CREATE TEMPORARY TABLE temp_slotdefault (id INTEGER PRIMARY KEY, slotoption INTEGER NOT NULL)' + + DROP_TEMPORARY_DEVICEREFERENCE = 'DROP TABLE IF EXISTS temp_devicereference' + DROP_TEMPORARY_SLOTOPTION = 'DROP TABLE IF EXISTS temp_slotoption' + DROP_TEMPORARY_SLOTDEFAULT = 'DROP TABLE IF EXISTS temp_slotdefault' + INDEX_MACHINE_ISDEVICE_SHORTNAME = 'CREATE INDEX machine_isdevice_shortname ON machine (isdevice ASC, shortname ASC)' INDEX_MACHINE_ISDEVICE_DESCRIPTION = 'CREATE INDEX machine_isdevice_description ON machine (isdevice ASC, description ASC)' INDEX_MACHINE_RUNNABLE_SHORTNAME = 'CREATE INDEX machine_runnable_shortname ON machine (runnable ASC, shortname ASC)' @@ -19,8 +27,6 @@ class SchemaQueries(object): INDEX_CLONEOF_PARENT = 'CREATE INDEX cloneof_parent ON cloneof (parent ASC)' - INDEX_DEVICEREFERENCE_DEVICE = 'CREATE INDEX devicereference_device ON devicereference (device ASC)' - INDEX_DIPSWITCH_MACHINE_ISCONFIG = 'CREATE INDEX dipswitch_machine_isconfig ON dipswitch (machine ASC, isconfig ASC)' DROP_MACHINE_ISDEVICE_SHORTNAME = 'DROP INDEX IF EXISTS machine_isdevice_shortname' @@ -35,8 +41,6 @@ class SchemaQueries(object): DROP_CLONEOF_PARENT = 'DROP INDEX IF EXISTS cloneof_parent' - DROP_DEVICEREFERENCE_DEVICE = 'DROP INDEX IF EXISTS devicereference_device' - DROP_DIPSWITCH_MACHINE_ISCONFIG = 'DROP INDEX IF EXISTS dipswitch_machine_isconfig' @@ -47,11 +51,19 @@ class UpdateQueries(object): ADD_SYSTEM = 'INSERT INTO system (id, year, manufacturer) VALUES (?, ?, ?)' ADD_CLONEOF = 'INSERT INTO cloneof (id, parent) VALUES (?, ?)' ADD_ROMOF = 'INSERT INTO romof (id, parent) VALUES (?, ?)' - ADD_DEVICEREFERENCE = 'INSERT OR IGNORE INTO devicereference (machine, device) VALUES (?, ?)' ADD_DIPSWITCH = 'INSERT INTO dipswitch (machine, isconfig, name, tag, mask) VALUES (?, ?, ?, ?, ?)' ADD_DIPLOCATION = 'INSERT INTO diplocation (dipswitch, bit, name, num, inverted) VALUES (?, ?, ?, ?, ?)' ADD_DIPVALUE = 'INSERT INTO dipvalue (dipswitch, name, value, isdefault) VALUES (?, ?, ?, ?)' ADD_FEATURE = 'INSERT INTO feature (machine, featuretype, status, overall) SELECT ?, id, ?, ? FROM featuretype WHERE name = ?' + ADD_SLOT = 'INSERT INTO slot (machine, name) VALUES (?, ?)' + + ADD_TEMPORARY_DEVICEREFERENCE = 'INSERT OR IGNORE INTO temp_devicereference (machine, device) VALUES (?, ?)' + ADD_TEMPORARY_SLOTOPTION = 'INSERT INTO temp_slotoption (slot, device, name) VALUES (?, ?, ?)' + ADD_TEMPORARY_SLOTDEFAULT = 'INSERT INTO temp_slotdefault (id, slotoption) VALUES (?, ?)' + + FINALISE_DEVICEREFERENCES = 'INSERT INTO devicereference (id, machine, device) SELECT temp_devicereference.id, temp_devicereference.machine, machine.id FROM temp_devicereference LEFT JOIN machine ON temp_devicereference.device = machine.shortname' + FINALISE_SLOTOPTIONS = 'INSERT INTO slotoption (id, slot, device, name) SELECT temp_slotoption.id, temp_slotoption.slot, machine.id, temp_slotoption.name FROM temp_slotoption LEFT JOIN machine ON temp_slotoption.device = machine.shortname' + FINALISE_SLOTDEFAULTS = 'INSERT INTO slotdefault (id, slotoption) SELECT id, slotoption FROM temp_slotdefault' class QueryCursor(object): @@ -140,21 +152,21 @@ class QueryCursor(object): return self.dbcurs.execute( 'SELECT shortname, description ' \ 'FROM machine ' \ - 'WHERE id IN (SELECT machine FROM devicereference WHERE device IN (SELECT shortname FROM machine WHERE sourcefile IN (SELECT id FROM sourcefile WHERE filename GLOB ?))) AND runnable = 1 ' \ + 'WHERE id IN (SELECT machine FROM devicereference WHERE device IN (SELECT id FROM machine WHERE sourcefile IN (SELECT id FROM sourcefile WHERE filename GLOB ?))) AND runnable = 1 ' \ 'ORDER BY shortname ASC', patterns) elif self.is_glob(*patterns): return self.dbcurs.execute( 'SELECT shortname, description ' \ 'FROM machine ' \ - 'WHERE id IN (SELECT machine FROM devicereference WHERE device IN (SELECT shortname FROM machine WHERE sourcefile IN (SELECT id FROM sourcefile WHERE filename GLOB ?' + (' OR filename GLOB ?' * (len(patterns) - 1)) + '))) AND runnable = 1 ' \ + 'WHERE id IN (SELECT machine FROM devicereference WHERE device IN (SELECT id FROM machine WHERE sourcefile IN (SELECT id FROM sourcefile WHERE filename GLOB ?' + (' OR filename GLOB ?' * (len(patterns) - 1)) + '))) AND runnable = 1 ' \ 'ORDER BY shortname ASC', patterns) else: return self.dbcurs.execute( 'SELECT shortname, description ' \ 'FROM machine ' \ - 'WHERE id IN (SELECT machine FROM devicereference WHERE device IN (SELECT shortname FROM machine WHERE sourcefile IN (SELECT id FROM sourcefile WHERE filename IN (?' + (', ?' * (len(patterns) - 1)) + ')))) AND runnable = 1 ' \ + 'WHERE id IN (SELECT machine FROM devicereference WHERE device IN (SELECT id FROM machine WHERE sourcefile IN (SELECT id FROM sourcefile WHERE filename IN (?' + (', ?' * (len(patterns) - 1)) + ')))) AND runnable = 1 ' \ 'ORDER BY shortname ASC', patterns) @@ -167,17 +179,17 @@ class QueryCursor(object): def get_devices_referenced(self, machine): return self.dbcurs.execute( - 'SELECT devicereference.device AS shortname, machine.description AS description, sourcefile.filename AS sourcefile ' \ - 'FROM devicereference LEFT JOIN machine ON devicereference.device = machine.shortname LEFT JOIN sourcefile ON machine.sourcefile = sourcefile.id ' \ + 'SELECT machine.shortname AS shortname, machine.description AS description, sourcefile.filename AS sourcefile ' \ + 'FROM devicereference LEFT JOIN machine ON devicereference.device = machine.id LEFT JOIN sourcefile ON machine.sourcefile = sourcefile.id ' \ 'WHERE devicereference.machine = ?', (machine, )) - def get_device_references(self, shortname): + def get_device_references(self, device): return self.dbcurs.execute( 'SELECT machine.shortname AS shortname, machine.description AS description, sourcefile.filename AS sourcefile ' \ 'FROM machine JOIN sourcefile ON machine.sourcefile = sourcefile.id ' \ 'WHERE machine.id IN (SELECT machine FROM devicereference WHERE device = ?)', - (shortname, )) + (device, )) def get_sourcefile_id(self, filename): return (self.dbcurs.execute('SELECT id FROM sourcefile WHERE filename = ?', (filename, )).fetchone() or (None, ))[0] @@ -241,7 +253,7 @@ class UpdateCursor(object): return self.dbcurs.lastrowid def add_devicereference(self, machine, device): - self.dbcurs.execute(UpdateQueries.ADD_DEVICEREFERENCE, (machine, device)) + self.dbcurs.execute(UpdateQueries.ADD_TEMPORARY_DEVICEREFERENCE, (machine, device)) def add_dipswitch(self, machine, isconfig, name, tag, mask): self.dbcurs.execute(UpdateQueries.ADD_DIPSWITCH, (machine, isconfig, name, tag, mask)) @@ -259,6 +271,18 @@ class UpdateCursor(object): self.dbcurs.execute(UpdateQueries.ADD_FEATURE, (machine, status, overall, featuretype)) return self.dbcurs.lastrowid + def add_slot(self, machine, name): + self.dbcurs.execute(UpdateQueries.ADD_SLOT, (machine, name)) + return self.dbcurs.lastrowid + + def add_slotoption(self, slot, device, name): + self.dbcurs.execute(UpdateQueries.ADD_TEMPORARY_SLOTOPTION, (slot, device, name)) + return self.dbcurs.lastrowid + + def add_slotdefault(self, slot, slotoption): + self.dbcurs.execute(UpdateQueries.ADD_TEMPORARY_SLOTDEFAULT, (slot, slotoption)) + return self.dbcurs.lastrowid + class QueryConnection(object): def __init__(self, database, **kwargs): @@ -293,6 +317,26 @@ class UpdateConnection(object): def cursor(self): return UpdateCursor(self.dbconn) + def prepare_for_load(self): + self.drop_indexes() + self.dbconn.execute(SchemaQueries.CREATE_TEMPORARY_DEVICEREFERENCE) + self.dbconn.execute(SchemaQueries.CREATE_TEMPORARY_SLOTOPTION) + self.dbconn.execute(SchemaQueries.CREATE_TEMPORARY_SLOTDEFAULT) + self.dbconn.commit() + + def finalise_load(self): + self.dbconn.execute(UpdateQueries.FINALISE_DEVICEREFERENCES) + self.dbconn.commit() + self.dbconn.execute(SchemaQueries.DROP_TEMPORARY_DEVICEREFERENCE) + self.dbconn.execute(UpdateQueries.FINALISE_SLOTOPTIONS) + self.dbconn.commit() + self.dbconn.execute(SchemaQueries.DROP_TEMPORARY_SLOTOPTION) + self.dbconn.execute(UpdateQueries.FINALISE_SLOTDEFAULTS) + self.dbconn.commit() + self.dbconn.execute(SchemaQueries.DROP_TEMPORARY_SLOTDEFAULT) + self.create_indexes() + self.dbconn.commit() + def drop_indexes(self): self.dbconn.execute(SchemaQueries.DROP_MACHINE_ISDEVICE_SHORTNAME) self.dbconn.execute(SchemaQueries.DROP_MACHINE_ISDEVICE_DESCRIPTION) @@ -302,9 +346,7 @@ class UpdateConnection(object): self.dbconn.execute(SchemaQueries.DROP_SYSTEM_MANUFACTURER) self.dbconn.execute(SchemaQueries.DROP_ROMOF_PARENT) self.dbconn.execute(SchemaQueries.DROP_CLONEOF_PARENT) - self.dbconn.execute(SchemaQueries.DROP_DEVICEREFERENCE_DEVICE) self.dbconn.execute(SchemaQueries.DROP_DIPSWITCH_MACHINE_ISCONFIG) - self.dbconn.commit() def create_indexes(self): self.dbconn.execute(SchemaQueries.INDEX_MACHINE_ISDEVICE_SHORTNAME) @@ -315,6 +357,4 @@ class UpdateConnection(object): self.dbconn.execute(SchemaQueries.INDEX_SYSTEM_MANUFACTURER) self.dbconn.execute(SchemaQueries.INDEX_ROMOF_PARENT) self.dbconn.execute(SchemaQueries.INDEX_CLONEOF_PARENT) - self.dbconn.execute(SchemaQueries.INDEX_DEVICEREFERENCE_DEVICE) self.dbconn.execute(SchemaQueries.INDEX_DIPSWITCH_MACHINE_ISCONFIG) - self.dbconn.commit() diff --git a/scripts/minimaws/lib/lxparse.py b/scripts/minimaws/lib/lxparse.py index f9061bcb798..cf194128708 100644 --- a/scripts/minimaws/lib/lxparse.py +++ b/scripts/minimaws/lib/lxparse.py @@ -134,7 +134,32 @@ class DipSwitchHandler(ElementHandler): self.setChildHandler(name, attrs, self.IGNORE) +class SlotHandler(ElementHandler): + def __init__(self, parent, **kwargs): + super(SlotHandler, self).__init__(parent=parent, **kwargs) + self.dbcurs = parent.dbcurs + self.machine = parent.id + + def startMainElement(self, name, attrs): + self.id = self.dbcurs.add_slot(self.machine, attrs['name']) + + def startChildElement(self, name, attrs): + if name == 'slotoption': + option = self.dbcurs.add_slotoption(self.id, attrs['devname'], attrs['name']) + if attrs.get('default') == 'yes': + self.dbcurs.add_slotdefault(self.id, option) + self.setChildHandler(name, attrs, self.IGNORE) + + class MachineHandler(ElementHandler): + CHILD_HANDLERS = { + 'description': TextAccumulator, + 'year': TextAccumulator, + 'manufacturer': TextAccumulator, + 'dipswitch': DipSwitchHandler, + 'configuration': DipSwitchHandler, + 'slot': SlotHandler } + def __init__(self, parent, **kwargs): super(MachineHandler, self).__init__(parent=parent, **kwargs) self.dbcurs = self.dbconn.cursor() @@ -149,10 +174,8 @@ class MachineHandler(ElementHandler): self.dbcurs.add_sourcefile(self.sourcefile) def startChildElement(self, name, attrs): - if (name == 'description') or (name == 'year') or (name == 'manufacturer'): - self.setChildHandler(name, attrs, TextAccumulator(self)) - elif (name == 'dipswitch') or (name == 'configuration'): - self.setChildHandler(name, attrs, DipSwitchHandler(self)) + if name in self.CHILD_HANDLERS: + self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) else: if name == 'device_ref': self.dbcurs.add_devicereference(self.id, attrs['name']) @@ -178,7 +201,6 @@ class MachineHandler(ElementHandler): self.dbcurs.add_system(self.id, self.year, self.manufacturer) def endMainElement(self, name): - self.dbconn.commit() self.dbcurs.close() @@ -199,11 +221,12 @@ class ListXmlHandler(ElementHandler): msg=('Expected "mame" element but found "%s"' % (name, )), exception=None, locator=self.locator) - self.dbconn.drop_indexes() + self.dbconn.prepare_for_load() + self.machines = 0 def endMainElement(self, name): # TODO: build index by first letter or whatever - self.dbconn.create_indexes() + self.dbconn.finalise_load() def startChildElement(self, name, attrs): if name != 'machine': @@ -213,6 +236,14 @@ class ListXmlHandler(ElementHandler): locator=self.locator) self.setChildHandler(name, attrs, MachineHandler(self)) + def endChildHandler(self, name, handler): + if name == 'machine': + if self.machines >= 1023: + self.dbconn.commit() + self.machines = 0 + else: + self.machines += 1 + def processingInstruction(self, target, data): pass diff --git a/scripts/minimaws/lib/wsgiserve.py b/scripts/minimaws/lib/wsgiserve.py index 482c9583b4e..96d03de4cfe 100644 --- a/scripts/minimaws/lib/wsgiserve.py +++ b/scripts/minimaws/lib/wsgiserve.py @@ -126,6 +126,7 @@ class MachineHandler(QueryPageHandler): return self.machine_page(machine_info) def machine_page(self, machine_info): + id = machine_info['id'] description = machine_info['description'] yield htmltmpl.MACHINE_PROLOGUE.substitute( assets=cgi.escape(urlparse.urljoin(self.application_uri, 'static'), True), @@ -163,7 +164,7 @@ class MachineHandler(QueryPageHandler): yield '</table>\n'.encode('utf-8') first = True - for name, desc, src in self.dbcurs.get_devices_referenced(machine_info['id']): + for name, desc, src in self.dbcurs.get_devices_referenced(id): if first: yield \ '<h2>Devices Referenced</h2>\n' \ @@ -178,7 +179,7 @@ class MachineHandler(QueryPageHandler): yield ' </tbody>\n</table>\n<script>make_table_sortable(document.getElementById("tbl-dev-refs"));</script>\n'.encode('utf-8') first = True - for name, desc, src in self.dbcurs.get_device_references(self.shortname): + for name, desc, src in self.dbcurs.get_device_references(id): if first: yield \ '<h2>Referenced By</h2>\n' \ diff --git a/scripts/minimaws/schema.sql b/scripts/minimaws/schema.sql index 7633441a123..86cec49cbcb 100644 --- a/scripts/minimaws/schema.sql +++ b/scripts/minimaws/schema.sql @@ -41,9 +41,10 @@ CREATE TABLE romof ( CREATE TABLE devicereference ( id INTEGER PRIMARY KEY, machine INTEGER NOT NULL, - device TEXT NOT NULL, + device INTEGER NOT NULL, UNIQUE (machine ASC, device ASC), - FOREIGN KEY (machine) REFERENCES machine (id)); + FOREIGN KEY (machine) REFERENCES machine (id), + FOREIGN KEY (device) REFERENCES machine (id)); CREATE TABLE dipswitch ( id INTEGER PRIMARY KEY, @@ -82,3 +83,25 @@ CREATE TABLE feature ( UNIQUE (machine ASC, featuretype ASC), FOREIGN KEY (machine) REFERENCES machine (id), FOREIGN KEY (featuretype) REFERENCES featuretype (id)); + +CREATE TABLE slot ( + id INTEGER PRIMARY KEY, + machine INTEGER NOT NULL, + name TEXT NOT NULL, + UNIQUE (machine ASC, name ASC), + FOREIGN KEY (machine) REFERENCES machine (id)); + +CREATE TABLE slotoption ( + id INTEGER PRIMARY KEY, + slot INTEGER NOT NULL, + device INTEGER NOT NULL, + name TEXT NOT NULL, + UNIQUE (slot ASC, name ASC), + FOREIGN KEY (slot) REFERENCES slot (id), + FOREIGN KEY (device) REFERENCES machine (id)); + +CREATE TABLE slotdefault ( + id INTEGER PRIMARY KEY, + slotoption INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES slot (id), + FOREIGN KEY (slotoption) REFERENCES slotoption (id)); |