diff options
author | 2022-06-25 04:57:24 +1000 | |
---|---|---|
committer | 2022-06-25 04:57:24 +1000 | |
commit | 923ef2c25db75ccb6a6d747304fc9c79ef454a46 (patch) | |
tree | 1a53bc7898adc2a7015a5a9e208c5df8cb024023 /plugins/data | |
parent | 2f453da00cab66472bde7bc40ca22ce7f4b89e74 (diff) |
-plugins.data: Reworked the code substantialy.
* Use the plugin data folder for storing the cache. The history folder
may be read-only or shared with different configurations.
* Don't create the cache database or surrounding folder if there's
nothing to store in it.
* Actually use prepared queries multiple times rather than always
destroying them after a single use.
* Added proper error checking for most database operations.
* Improved query performance by avoiding outer joins and table scans.
-bus/nubus: Made the Macintosh Display Cards map the blue channel to
white with monochrome monitors. Also added logging for PLL
configuration to help debug how CRTC and RAMDAC clocks work in the
future.
Diffstat (limited to 'plugins/data')
-rw-r--r-- | plugins/data/data_command.lua | 13 | ||||
-rw-r--r-- | plugins/data/data_gameinit.lua | 10 | ||||
-rw-r--r-- | plugins/data/data_history.lua | 247 | ||||
-rw-r--r-- | plugins/data/data_mameinfo.lua | 20 | ||||
-rw-r--r-- | plugins/data/data_marp.lua | 182 | ||||
-rw-r--r-- | plugins/data/data_messinfo.lua | 19 | ||||
-rw-r--r-- | plugins/data/data_story.lua | 19 | ||||
-rw-r--r-- | plugins/data/data_sysinfo.lua | 9 | ||||
-rw-r--r-- | plugins/data/database.lua | 170 | ||||
-rw-r--r-- | plugins/data/init.lua | 20 | ||||
-rw-r--r-- | plugins/data/load_dat.lua | 336 |
11 files changed, 661 insertions, 384 deletions
diff --git a/plugins/data/data_command.lua b/plugins/data/data_command.lua index cd265d25b79..910a20beadc 100644 --- a/plugins/data/data_command.lua +++ b/plugins/data/data_command.lua @@ -1,13 +1,16 @@ local dat = {} + local info, ver -local datread = require("data/load_dat") +local datread = require('data/load_dat') do local buttonchar local function convert(str) - if not buttonchar then buttonchar = require("data/button_char") end + if not buttonchar then + buttonchar = require("data/button_char") + end return buttonchar(str) end - datread, ver = datread.open("command.dat", "#[^V]*Ver[^.:]*[.:]", convert) + datread, ver = datread.open('command.dat', '#[^V]*Ver[^.:]*[.:]', convert) end function dat.check(set, softlist) @@ -15,11 +18,11 @@ function dat.check(set, softlist) return nil end local status - status, info = pcall(datread, "cmd", "info", set) + status, info = pcall(datread, 'cmd', 'info', set) if not status or not info then return nil end - return _p("plugin-data", "Command") + return _p('plugin-data', 'Command') end function dat.get() diff --git a/plugins/data/data_gameinit.lua b/plugins/data/data_gameinit.lua index 31e659325c4..f6856629bb8 100644 --- a/plugins/data/data_gameinit.lua +++ b/plugins/data/data_gameinit.lua @@ -1,19 +1,19 @@ local dat = {} -local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("gameinit.dat", "# .-GAMEINIT.DAT") +local ver, info +local datread = require('data/load_dat') +datread, ver = datread.open('gameinit.dat', '# .-GAMEINIT.DAT') function dat.check(set, softlist) if softlist or not datread then return nil end local status - status, info = pcall(datread, "mame", "info", set) + status, info = pcall(datread, 'mame', 'info', set) if not status or not info then return nil end - return _p("plugin-data", "Gameinit") + return _p('plugin-data', 'Gameinit') end function dat.get() diff --git a/plugins/data/data_history.lua b/plugins/data/data_history.lua index e6fd4eb2509..b233405e984 100644 --- a/plugins/data/data_history.lua +++ b/plugins/data/data_history.lua @@ -1,149 +1,178 @@ local dat = {} -local db = require("data/database") + +local db = require('data/database') local ver, info -local file = "history.xml" +local file = 'history.xml' +local tablename local function init() - local filepath - local dbver - local fh - - for path in mame_manager.ui.options.entries.historypath:value():gmatch("([^;]+)") do - filepath = emu.subst_env(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end - -- check for old history table - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - stmt:bind_values("history.dat") - if stmt:step() == db.ROW then - stmt:finalize() - db.exec("DROP TABLE \"history.dat\"") - db.exec("DROP TABLE \"history.dat_idx\"") - stmt = db.prepare("DELETE FROM version WHERE datfile = ?") - stmt:bind_values("history.dat") - stmt:step() + if db.get_version('history.dat') then + db.exec([[DROP TABLE "history.dat";]]) + db.exec([[DROP TABLE "history.dat_idx";]]) + db.set_version('history.dat', nil) end - stmt:finalize() - - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading history version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) - end - stmt:finalize() - - if not fh and dbver then - -- data in database but missing file, just use what we have - ver = dbver - return - elseif not fh then + local fh, filepath, dbver + fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + ver = dbver + end return - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[_idx" ( - name VARCHAR NOT NULL, - list VARCHAR, - data INTEGER NOT NULL)]]) - db.check("creating index") - db.exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)") - db.check("creating table") - db.exec("CREATE INDEX \"name_" .. file .. "\" ON \"" .. file .. "_idx\"(name)") - db.check("creating system index") end + -- scan file for version for line in fh:lines() do - local match = line:match("<history([^>]*)>") + local match = line:match('<history([^>]*)>') if match then - match = match:match("version=\"([^\"]*)\"") + match = match:match('version="([^"]*)"') if match then ver = match break end end end - - if not ver then + if (not ver) or (ver == dbver) then fh:close() + ver = dbver return end - if ver == dbver then + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s_idx" ( + name VARCHAR NOT NULL, + list VARCHAR NOT NULL, + data INTEGER NOT NULL);]], + tablename)) + db.check(string.format('creating %s index table', file)) + db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename)) + db.check(string.format('creating %s data table', file)) + db.exec( + string.format( + [[CREATE INDEX "namelist_%s" ON "%s_idx" (name, list);]], + tablename, tablename)) + db.check(string.format('creating %s name/list index', file)) + end + + local slaxml = require('xml') + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check(string.format('starting %s transaction', file)) then fh:close() + ver = dbver return end + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting history") - db.exec("DELETE FROM \"" .. file .. "_idx\"") - db.check("deleting index") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating history version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting history version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + db.exec(string.format([[DELETE FROM "%s_idx";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + end + db.set_version(file, ver) + if not db.check(string.format('updating %s version', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning history transation") + fh:seek('set') + local buffer = fh:read('a') local lasttag local entry = {} local rowid - local slaxml = require("xml") + + local dataquery = db.prepare( + string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename)) + local indexquery = db.prepare( + string.format([[INSERT INTO "%s_idx" (name, list, data) VALUES (?, ?, ?);]], tablename)) local parser = slaxml:parser{ startElement = function(name) lasttag = name - if name == "entry" then + if name == 'entry' then entry = {} rowid = nil - elseif (name == "system") or (name == "item") then - entry[#entry + 1] = {} + elseif (name == 'system') or (name == 'item') then + table.insert(entry, {}) end end, attribute = function(name, value) - if (name == "name") or (name == "list") then + if (name == 'name') or (name == 'list') then entry[#entry][name] = value end end, text = function(text, cdata) - if lasttag == "text" then - text = text:gsub("\r", "") -- strip crs - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?)") - db.check("inserting values") - stmt:bind_values(text) - stmt:step() - rowid = stmt:last_insert_rowid() - stmt:finalize() + if lasttag == 'text' then + text = text:gsub('\r', '') -- strip carriage returns + dataquery:bind_values(text) + while true do + local status = dataquery:step() + if status == db.DONE then + rowid = dataquery:last_insert_rowid(); + break + elseif result == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + -- FIXME: how to abort parse and roll back? + break + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + dataquery:reset() end end, closeElement = function(name) - if name == "entry" then + if (name == 'entry') and rowid then for num, entry in pairs(entry) do - stmt = db.prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?)") - db.check("inserting into index") - stmt:bind_values(entry.name, entry.list, rowid) - stmt:step() - stmt:finalize() + indexquery:bind_values(entry.name, entry.list or '', rowid) + while true do + local status = indexquery:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + -- FIXME: how to abort parse and roll back? + break + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + indexquery:reset() end end end } - parser:parse(buffer, {stripWhitespace=true}) + + parser:parse(buffer, { stripWhitespace = true }) + dataquery:finalize() + indexquery:finalize() fh:close() - db.exec("END TRANSACTION") - db.check("ending history transation") + + db.exec([[COMMIT TRANSACTION;]]) + if not db.check(string.format('committing %s transaction', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + ver = dbver + end end if db then @@ -151,25 +180,31 @@ if db then end function dat.check(set, softlist) - if not ver or not db then + if not ver then return nil end + info = nil - local stmt - if softlist then - stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [[" - AS f WHERE fi.name = ? AND fi.list = ? AND f.rowid = fi.data]]) - stmt:bind_values(set, softlist) - else - stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [[" - AS f WHERE fi.name = ? AND fi.list ISNULL AND f.rowid = fi.data]]) - stmt:bind_values(set) - end - if stmt:step() == db.ROW then - info = stmt:get_value(0) + local query = db.prepare( + string.format( + [[SELECT f.data + FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid + WHERE fi.name = ? AND fi.list = ?;]], + tablename, tablename)) + query:bind_values(set, softlist or '') + while not info do + local status = query:step() + if status == db.ROW then + info = query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check(string.format('reading %s data', file)) + break + end end - stmt:finalize() - return info and _p("plugin-data", "History") or nil + query:finalize() + return info and _p('plugin-data', 'History') or nil end function dat.get() diff --git a/plugins/data/data_mameinfo.lua b/plugins/data/data_mameinfo.lua index 7a6e6ea2def..0e77e328cef 100644 --- a/plugins/data/data_mameinfo.lua +++ b/plugins/data/data_mameinfo.lua @@ -1,23 +1,27 @@ local dat = {} -local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("mameinfo.dat", "# MAMEINFO.DAT", function(str) return str:gsub("\n\n", "\n") end) + +local info, ver +local datread = require('data/load_dat') +datread, ver = datread.open( + 'mameinfo.dat', + '# MAMEINFO.DAT', + function (str) return str:gsub('\n\n', '\n') end) function dat.check(set, softlist) if softlist or not datread then return nil end local status, drvinfo - status, info = pcall(datread, "mame", "info", set) + status, info = pcall(datread, 'mame', 'info', set) if not status or not info then return nil end - local sourcefile = emu.driver_find(set).source_file:match("[^/\\]*$") - status, drvinfo = pcall(datread, "drv", "info", sourcefile) + local sourcefile = emu.driver_find(set).source_file:match('[^/\\]*$') + status, drvinfo = pcall(datread, 'drv', 'info', sourcefile) if drvinfo then - info = info .. _p("plugin-data", "\n\n--- DRIVER INFO ---\nDriver: ") .. sourcefile .. "\n\n" .. drvinfo + info = info .. _p('plugin-data', '\n\n--- DRIVER INFO ---\nDriver: ') .. sourcefile .. '\n\n' .. drvinfo end - return _p("plugin-data", "MAMEinfo") + return _p('plugin-data', 'MAMEinfo') end function dat.get() diff --git a/plugins/data/data_marp.lua b/plugins/data/data_marp.lua index 47a45785a81..09fddf11ef9 100644 --- a/plugins/data/data_marp.lua +++ b/plugins/data/data_marp.lua @@ -1,120 +1,144 @@ -- get marp high score file from http://replay.marpirc.net/txt/scores3.htm local dat = {} + local db = require("data/database") local ver, info local function init() - local filepath - local dbver - local fh - local file = "scores3.htm" - - for path in mame_manager.ui.options.entries.historypath:value():gmatch("([^;]+)") do - filepath = emu.subst_env(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end + local file = 'scores3.htm' - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading marp version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) - end - stmt:finalize() - - if not fh and dbver then - -- data in database but missing file, just use what we have - ver = dbver - return - elseif not fh then + local fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + ver = dbver + end return - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[" ( - romset VARCHAR NOT NULL, - data CLOB NOT NULL)]]) - db.check("creating marp table") end + -- scan file for version for line in fh:lines() do - local match = line:match("Top Scores from the MAME Action Replay Page %(([%w :]+)%)") + local match = line:match('Top Scores from the MAME Action Replay Page %(([%w :]+)%)') if match then ver = match break end end - - if not ver then + if (not ver) or (ver == dbver) then fh:close() + ver = dbver return end - if ver == dbver then + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s" ( + romset VARCHAR NOT NULL, + data CLOB NOT NULL, + PRIMARY KEY(romset));]], + tablename)) + db.check('creating MARP table') + end + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check('starting MARP transaction') then fh:close() + ver = dbver return end + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting marp") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating marp version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting marp version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check('deleting previous MARP data') then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + end + if not db.set_version(file, ver) then + db.check('updating MARP version') + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning marp transation") + fh:seek('set') + local buffer = fh:read('a') local function gmatchpos() local pos = 1 - local set, data = "" + local set = '' + local data = '' local function iter() local lastset = set while true do local spos, scr, plyr, stype, ltype - local url = "" - spos, pos, set, stype, scr, plyr, ltype = buffer:find("\n%s*([%w_]*)%-?(%w-) :%s*(%d+) [;:] ([^:]+): [^%[\n]*%[?([%w ]*)[^\n]*", pos) + spos, pos, set, stype, scr, plyr, ltype = buffer:find('\n%s*([%w_]*)%-?(%w-) :%s*(%d+) [;:] ([^:]+): [^%[\n]*%[?([%w ]*)[^\n]*', pos) if not spos then return nil end - if set ~= "" then - if ltype ~= "" then - url = ltype .. "\t\n" - else - url = "" + + local url = '' + if set ~= '' then + if ltype ~= '' then + url = ltype .. '\t\n' + end + url = url .. 'http://replay.marpirc.net/r/' .. set + if stype ~= '' then + url = url .. '-' .. stype end - url = url .. "http://replay.marpirc.net/r/" .. set .. ((stype ~= "") and ("-" .. stype) or "") .. "\t\n" + url = url .. '\t\n' end - if set ~= "" and lastset and lastset ~= set then + + if (set ~= '') and (lastset ~= set) then local lastdata = data - data = url .. plyr .. "\t" .. scr .. "\n" + data = url .. plyr .. '\t' .. scr .. '\n' return lastset, lastdata end - data = data .. ((url ~= "") and ("\n" .. url) or "") .. plyr .. "\t" .. scr .. "\n" + + if url ~= '' then + data = data .. '\n' .. url + end + data = data .. plyr .. '\t' .. scr .. '\n' end end return iter end + local query = db.prepare( + string.format([[INSERT INTO "%s" (romset, data) VALUES (?, ?)]], tablename)) for set, data in gmatchpos() do - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?, ?)") - db.check("inserting marp values") - stmt:bind_values(set, data) - stmt:step() - stmt:finalize() + query:bind_values(set, data) + while true do + local status = query:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error('Database busy: inserting MARP data') + query:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + elseif result ~= db.ROW then + db.check('inserting MARP data') + break + end + end + query:reset() end + query:finalize() + fh:close() - db.exec("END TRANSACTION") - db.check("ending marp transation") + db.exec([[COMMIT TRANSACTION;]]) + if not db.check('committing MARP transaction') then + db.exec([[ROLLBACK TRANSACTION;]]) + ver = dbver + end end if db then @@ -122,18 +146,26 @@ if db then end function dat.check(set, softlist) - if softlist or not ver or not db then + if softlist or (not ver) then return nil end + info = nil - local stmt = db.prepare("SELECT data FROM \"scores3.htm\" AS f WHERE romset = ?") - db.check("reading marp data") - stmt:bind_values(set) - if stmt:step() == db.ROW then - info = "#j2\n" .. stmt:get_value(0) + local query = db.prepare([[SELECT data FROM "scores3.htm" WHERE romset = ?;]]) + query:bind_values(set) + while not info do + local status = query:step() + if status == db.ROW then + info = '#j2\n' .. query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check('reading MARP data') + break + end end - stmt:finalize() - return info and _p("plugin-data", "MARPScore") or nil + query:finalize() + return info and _p('plugin-data', 'MARPScore') or nil end function dat.get() diff --git a/plugins/data/data_messinfo.lua b/plugins/data/data_messinfo.lua index fe2099890ee..3c94322474f 100644 --- a/plugins/data/data_messinfo.lua +++ b/plugins/data/data_messinfo.lua @@ -1,24 +1,27 @@ local dat = {} -local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("messinfo.dat", "# MESSINFO.DAT", function(str) return str:gsub("\n\n", "\n") end) +local ver, info +local datread = require('data/load_dat') +datread, ver = datread.open( + 'messinfo.dat', + '# MESSINFO.DAT', + function (str) return str:gsub('\n\n', '\n') end) function dat.check(set, softlist) if softlist or not datread then return nil end local status, drvinfo - status, info = pcall(datread, "mame", "info", set) + status, info = pcall(datread, 'mame', 'info', set) if not status or not info then return nil end - local sourcefile = emu.driver_find(set).source_file:match("[^/\\]*$") - status, drvinfo = pcall(datread, "drv", "info", sourcefile) + local sourcefile = emu.driver_find(set).source_file:match('[^/\\]*$') + status, drvinfo = pcall(datread, 'drv', 'info', sourcefile) if drvinfo then - info = info .. _p("plugin-data", "\n\n--- DRIVER INFO ---\nDriver: ") .. sourcefile .. "\n\n" .. drvinfo + info = info .. _p('plugin-data', '\n\n--- DRIVER INFO ---\nDriver: ') .. sourcefile .. '\n\n' .. drvinfo end - return _p("plugin-data", "MESSinfo") + return _p('plugin-data', 'MESSinfo') end function dat.get() diff --git a/plugins/data/data_story.lua b/plugins/data/data_story.lua index 70bedd02df0..b17bb4f1ea1 100644 --- a/plugins/data/data_story.lua +++ b/plugins/data/data_story.lua @@ -1,25 +1,26 @@ local dat = {} + local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("story.dat", "# version") +local datread = require('data/load_dat') +datread, ver = datread.open('story.dat', '# version') function dat.check(set, softlist) if softlist or not datread then return nil end - local status, data = pcall(datread, "story", "info", set) + local status, data = pcall(datread, 'story', 'info', set) if not status or not data then return nil end local lines = {} - data = data:gsub("MAMESCORE records : ([^\n]+)", "MAMESCORE records :\t\n%1", 1) - for line in data:gmatch("[^\n]*") do - if not (#lines == 0 and line == "") and not (lines[#lines] == "" and line == "") then - lines[#lines + 1] = line:gsub("^(.-)_+([0-9.]+)$", "%1\t%2") + data = data:gsub('MAMESCORE records : ([^\n]+)', 'MAMESCORE records :\t\n%1', 1) + for line in data:gmatch('[^\n]*') do + if (line ~= '') or ((#lines ~= 0) and (lines[#lines] ~= '')) then + table.insert(lines, line:gsub('^(.-)_+([0-9.]+)$', '%1\t%2')) end end - info = "#j2\n" .. table.concat(lines, "\n") - return _p("plugin-data", "Mamescore") + info = '#j2\n' .. table.concat(lines, '\n') + return _p('plugin-data', 'Mamescore') end function dat.get() diff --git a/plugins/data/data_sysinfo.lua b/plugins/data/data_sysinfo.lua index cb2daaa03a0..1ba13605821 100644 --- a/plugins/data/data_sysinfo.lua +++ b/plugins/data/data_sysinfo.lua @@ -1,18 +1,19 @@ local dat = {} + local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("sysinfo.dat", "# This file was generated on") +local datread = require('data/load_dat') +datread, ver = datread.open('sysinfo.dat', '# This file was generated on') function dat.check(set, softlist) if softlist or not datread then return nil end local status - status, info = pcall(datread, "bio", "info", set) + status, info = pcall(datread, 'bio', 'info', set) if not status or not info then return nil end - return _p("plugin-data", "Sysinfo") + return _p('plugin-data', 'Sysinfo') end function dat.get() diff --git a/plugins/data/database.lua b/plugins/data/database.lua index be52f23ee63..e0249182409 100644 --- a/plugins/data/database.lua +++ b/plugins/data/database.lua @@ -1,41 +1,159 @@ -local sql = require("lsqlite3") -local datfile = {} +local sql = require('lsqlite3') local db -local function check_db(msg) +local function check_db_result(msg) if db:errcode() > sql.OK then - emu.print_error(string.format("Error: %s (%s - %s)", msg, db:errcode(), db:errmsg())) + emu.print_error(string.format('Error: %s (%s - %s)', msg, db:errcode(), db:errmsg())) + return false end + return true end -do - local dbpath = emu.subst_env(mame_manager.ui.options.entries.historypath:value():match("([^;]+)")) - db = sql.open(dbpath .. "/history.db") - if not db then - lfs.mkdir(dbpath) - db = sql.open(dbpath .. "/history.db") +local function settings_path() + return emu.subst_env(manager.machine.options.entries.homepath:value():match('([^;]+)')) .. '/data' +end + +local function check_version_table() + local found = false + db:exec( + [[SELECT COUNT(*) FROM sqlite_master WHERE name = 'version' AND type = 'table';]], + function (udata, cols, values, names) + found = tonumber(values[1]) > 0 + return 0 + end) + if check_db_result('checking for "version" table') and (not found) then + db:exec( + [[CREATE TABLE version ( + datfile VARCHAR NOT NULL, + version VARCHAR NOT NULL, + PRIMARY KEY (datfile));]]) + found = check_db_result('creating "version" table') + end + if not found then + db:close() + db = nil + end +end + +local function open_existing() + db = sql.open(settings_path() .. '/history.db', sql.OPEN_READWRITE) + if db then + check_version_table() + end + return db +end + +local function open_create() + -- first try to create settings directory + local dir = settings_path() + local attr = lfs.attributes(dir) + if (not attr) and (not lfs.mkdir(dir)) then + emu.print_error(string.format('Error creating data plugin settings directory "%s"', dir)) + elseif attr and (attr.mode ~= 'directory') then + emu.print_error(string.format('Error opening data plugin database: "%s" is not a directory', dir)) + else + -- now try to open the database + local dbpath = dir .. '/history.db' + db = sql.open(dbpath, sql.OPEN_READWRITE | sql.OPEN_CREATE) if not db then - emu.print_error("Unable to create history.db") - return false + emu.print_error(string.format('Error opening data plugin database "%s"', dbpath)) + else + check_version_table() end - check_db("opening database") end + return db end -if db then - local found = false - db:exec("select * from sqlite_master where name = 'version'", function() found = true return 0 end) - check_db("checking for 'version' table") - if not found then - db:exec([[ - CREATE TABLE version ( - version VARCHAR NOT NULL, - datfile VARCHAR UNIQUE NOT NULL)]]) - check_db("creating 'version' table") + +local dbtable = { + ROW = sql.ROW, + BUSY = sql.BUSY, + DONE = sql.DONE, + check = check_db_result } + +function dbtable.sanitize_name(name) + return name:gsub('[^%w%."]', '_') +end + +function dbtable.get_version(filename) + -- don't try to create the database here, just return nil if it doesn't exist + if (not db) and (not open_existing()) then + return nil + end + local query = db:prepare([[SELECT version FROM version WHERE datfile = ?;]]) + query:bind_values(filename) + local result + while result == nil do + local status = query:step() + if status == sql.ROW then + result = query:get_value(0) + elseif status ~= sql.BUSY then + break + end + end + query:finalize() + return result +end + +function dbtable.set_version(filename, version) + if (not db) and (not open_create()) then + return nil + end + local query + if version ~= nil then + query = db:prepare( + [[INSERT INTO version (datfile, version) VALUES (?1, ?2) + ON CONFLICT(datfile) DO UPDATE set version = ?2;]]) + query:bind_values(filename, version) + else + query = db:prepare([[DELETE FROM version WHERE datfile = ?1;]]) + query:bind_values(filename) + end + local result + while result == nil do + local status = query:step() + if status == sqlite3.DONE then + result = true + elseif result ~= sqlite3.ROW then + result = false + end + end + query:finalize() + return result +end + +function dbtable.prepare(...) + if (not db) and open_create() then + dbtable.prepare = function (...) return db:prepare(...) end + end + if db then + return db:prepare(...) + else + return nil end end -local dbtable = { prepare = function(...) return db:prepare(...) end, - exec = function(...) return db:exec(...) end, ROW = sql.ROW, check = check_db } +function dbtable.exec(...) + if (not db) and open_create() then + dbtable.exec = function (...) return db:exec(...) end + end + if db then + return db:exec(...) + else + return nil + end +end + +function dbtable.open_data_file(file) + local fh, filepath + for path in mame_manager.ui.options.entries.historypath:value():gmatch('([^;]+)') do + filepath = emu.subst_env(path) .. '/' .. file + fh = io.open(filepath, 'r') + if fh then + break + end + end + return fh, filepath, dbtable.sanitize_name(file), dbtable.get_version(file) +end -return db and dbtable or false +return dbtable diff --git a/plugins/data/init.lua b/plugins/data/init.lua index f58742e8d34..eb3ad1ae9b3 100644 --- a/plugins/data/init.lua +++ b/plugins/data/init.lua @@ -4,12 +4,12 @@ -- heading if it supports the set otherwise nil and get which returns the data -- the script should be named data_<name>.lua -- this is set default on in the plugin.json -local exports = {} -exports.name = "data" -exports.version = "0.0.1" -exports.description = "Data plugin" -exports.license = "BSD-3-Clause" -exports.author = { name = "Carl" } +local exports = { + name = 'data', + version = '0.0.2', + description = 'Data plugin', + license = 'BSD-3-Clause', + author = { name = 'Carl' } } local data = exports @@ -26,7 +26,7 @@ function data.startplugin() local cur_list emu.register_start( - function() + function () data_scr = {} for file in lfs.dir(plugindir) do local name = string.match(file, '^(data_.*).lua$') @@ -40,7 +40,7 @@ function data.startplugin() end) emu.register_callback( - function(set) + function (set) local ret = {} if set == '' then set = emu.romname() @@ -70,13 +70,13 @@ function data.startplugin() 'data_list') emu.register_callback( - function(num) + function (num) return valid_lst[num + 1].get() end, 'data') emu.register_callback( - function(num) + function (num) local ver if valid_lst[num + 1].ver then ver = valid_lst[num + 1].ver() diff --git a/plugins/data/load_dat.lua b/plugins/data/load_dat.lua index 7209872ad32..0438c5499ba 100644 --- a/plugins/data/load_dat.lua +++ b/plugins/data/load_dat.lua @@ -1,17 +1,29 @@ local datfile = {} -local db = require("data/database") -local function readret(file) - local function read(tag1, tag2, set) +local db = require('data/database') + +local function readret(file, tablename) + local query = db.prepare( + string.format( + [[SELECT f.data + FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid + WHERE fi.type = ? AND fi.val = ? AND fi.romset = ?;]], + tablename, tablename)) + local function read(tag, val, set) + query:bind_values(tag, val, set) local data - local stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [[" - AS f WHERE fi.type = ? AND fi.val = ? AND fi.romset = ? AND f.rowid = fi.data]]) - db.check("reading " .. tag1 .. " - " .. tag2 .. " - " .. set) - stmt:bind_values(tag1, tag2, set) - if stmt:step() == db.ROW then - data = stmt:get_value(0) + while not data do + local status = query:step() + if status == db.ROW then + data = query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check(string.format('reading %s data', file)) + break + end end - stmt:finalize() + query:reset() return data end return read @@ -22,49 +34,22 @@ function datfile.open(file, vertag, fixupcb) if not db then return nil end - local ver, dbver - local filepath - local fh - - for path in mame_manager.ui.options.entries.historypath:value():gmatch("([^;]+)") do - filepath = emu.subst_env(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end - -- remove unsafe chars from file for use in sql statement - file = file:gsub("[^%w%._]", "") - - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) - end - stmt:finalize() - if not fh and dbver then - -- data in database but missing file, just use what we have - return readret(file), dbver - elseif not fh then - return nil - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[_idx" ( - type VARCHAR NOT NULL, - val VARCHAR NOT NULL, - romset VARCHAR NOT NULL, - data INTEGER NOT NULL)]]) - db.check("creating index") - db.exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)") - db.check("creating table") - db.exec("CREATE INDEX \"typeval_" .. file .. "\" ON \"" .. file .. "_idx\"(type, val)") - db.check("creating typeval index") + local fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + return readret(file, tablename), dbver + else + return nil + end end + local ver if vertag then + -- scan file for version for line in fh:lines() do - local match = line:match(vertag .. "%s*(%S+)") + local match = line:match(vertag .. '%s*(%S+)') if match then ver = match break @@ -72,109 +57,204 @@ function datfile.open(file, vertag, fixupcb) end end if not ver then - -- use file ctime for version - ver = tostring(lfs.attributes(filepath, "change")) + -- fall back to file modification time for version + ver = tostring(lfs.attributes(filepath, 'change')) end if ver == dbver then fh:close() - return readret(file), dbver + return readret(file, tablename), dbver + end + + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s_idx" ( + type VARCHAR NOT NULL, + val VARCHAR NOT NULL, + romset VARCHAR NOT NULL, + data INTEGER NOT NULL);]], + tablename)) + db.check(string.format('creating %s index table', file)) + db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename)) + db.check(string.format('creating %s data table', file)) + db.exec( + string.format( + [[CREATE INDEX "typeval_%s" ON "%s_idx" (type, val, romset);]], + tablename, tablename)) + db.check(string.format('creating %s type/value index', file)) + end + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check(string.format('starting %s transaction', file)) then + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end end + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting") - db.exec("DELETE FROM \"" .. file .. "_idx\"") - db.check("deleting index") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end + db.exec(string.format([[DELETE FROM "%s_idx";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - - do - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning transaction") - local function gmatchpos() - local pos = 1 - local function iter() - local tags, data - while not data do - local npos - local spos, epos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", pos) - if not spos then - return nil - end - npos, epos = buffer:find("[\n\r]$%w+%s*[\n\r]+", epos) - if not npos then - return nil - end - tags = buffer:sub(spos, epos) - spos, npos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", epos) - if not spos then - return nil - end - data = buffer:sub(epos, spos) - pos = spos + db.set_version(file, ver) + if not db.check(string.format('updating %s version', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end + + local dataquery = db.prepare( + string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename)) + local indexquery = db.prepare( + string.format( + [[INSERT INTO "%s_idx" (type, val, romset, data) VALUES (?, ?, ?, ?)]], + tablename)) + + fh:seek('set') + local buffer = fh:read('a') + + local function gmatchpos() + local pos = 1 + local function iter() + local tags, data + while not data do + local npos + local spos, epos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', pos) + if not spos then + return nil + end + npos, epos = buffer:find('[\n\r]$%w+%s*[\n\r]+', epos) + if not npos then + return nil end - return tags, data + tags = buffer:sub(spos, epos) + spos, npos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', epos) + if not spos then + return nil + end + data = buffer:sub(epos, spos) + pos = spos end - return iter + return tags, data end - for info, data in gmatchpos() do - local tags = {} - local infotype - info = info:gsub(utf8.char(0xfeff), "") --remove boms - data = data:gsub(utf8.char(0xfeff), "") - for s in info:gmatch("[\n\r]$([^\n\r]*)") do - if s:find("=", 1, true) then - local m1, m2 = s:match("([^=]*)=(.*)") - for tag in m1:gmatch("[^,]+") do - for set in m2:gmatch("[^,]+") do - tags[#tags + 1] = { tag = tag, set = set } - end + return iter + end + + for info, data in gmatchpos() do + local tags = {} + local infotype + info = info:gsub(utf8.char(0xfeff), '') -- remove byte order marks + data = data:gsub(utf8.char(0xfeff), '') + for s in info:gmatch('[\n\r]$([^\n\r]*)') do + if s:find('=', 1, true) then + local m1, m2 = s:match('([^=]*)=(.*)') + for tag in m1:gmatch('[^,]+') do + for set in m2:gmatch('[^,]+') do + table.insert(tags, { tag = tag, set = set }) end - else - infotype = s - break end + else + infotype = s + break end + end + + data = data:gsub('[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].*', '') - data = data:gsub("[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]", "\n") - data = data:gsub("[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]", "\n") - data = data:gsub("[\n\r]$end%s*[\n\r].*", "") + if (#tags > 0) and infotype then + data = data:gsub('\r', '') -- strip carriage returns + if fixupcb then + data = fixupcb(data) + end - if #tags > 0 and infotype then - data = data:gsub("\r", "") -- strip crs - if fixupcb then - data = fixupcb(data) + dataquery:bind_values(data) + local row + while true do + local status = dataquery:step() + if status == db.DONE then + row = dataquery:last_insert_rowid(); + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break end - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?)") - db.check("inserting values") - stmt:bind_values(data) - stmt:step() - local row = stmt:last_insert_rowid() - stmt:finalize() + end + dataquery:reset() + + if row then for num, tag in pairs(tags) do - stmt = db.prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?, ?)") - db.check("inserting into index") - stmt:bind_values(infotype, tag.tag, tag.set, row) - stmt:step() - stmt:finalize() + indexquery:bind_values(infotype, tag.tag, tag.set, row) + while true do + local status = indexquery:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + indexquery:reset() end end end - db.exec("END TRANSACTION") - db.check("ending transaction") end + + dataquery:finalize() + indexquery:finalize() + fh:close() + db.exec([[COMMIT TRANSACTION;]]) + if not db.check(string.format('committing %s transaction', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end - return readret(file), ver + return readret(file, tablename), ver end return datfile |