diff options
author | 2022-06-25 04:57:24 +1000 | |
---|---|---|
committer | 2022-06-25 04:57:24 +1000 | |
commit | 923ef2c25db75ccb6a6d747304fc9c79ef454a46 (patch) | |
tree | 1a53bc7898adc2a7015a5a9e208c5df8cb024023 /plugins/data/load_dat.lua | |
parent | 2f453da00cab66472bde7bc40ca22ce7f4b89e74 (diff) |
-plugins.data: Reworked the code substantialy.
* Use the plugin data folder for storing the cache. The history folder
may be read-only or shared with different configurations.
* Don't create the cache database or surrounding folder if there's
nothing to store in it.
* Actually use prepared queries multiple times rather than always
destroying them after a single use.
* Added proper error checking for most database operations.
* Improved query performance by avoiding outer joins and table scans.
-bus/nubus: Made the Macintosh Display Cards map the blue channel to
white with monochrome monitors. Also added logging for PLL
configuration to help debug how CRTC and RAMDAC clocks work in the
future.
Diffstat (limited to 'plugins/data/load_dat.lua')
-rw-r--r-- | plugins/data/load_dat.lua | 336 |
1 files changed, 208 insertions, 128 deletions
diff --git a/plugins/data/load_dat.lua b/plugins/data/load_dat.lua index 7209872ad32..0438c5499ba 100644 --- a/plugins/data/load_dat.lua +++ b/plugins/data/load_dat.lua @@ -1,17 +1,29 @@ local datfile = {} -local db = require("data/database") -local function readret(file) - local function read(tag1, tag2, set) +local db = require('data/database') + +local function readret(file, tablename) + local query = db.prepare( + string.format( + [[SELECT f.data + FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid + WHERE fi.type = ? AND fi.val = ? AND fi.romset = ?;]], + tablename, tablename)) + local function read(tag, val, set) + query:bind_values(tag, val, set) local data - local stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [[" - AS f WHERE fi.type = ? AND fi.val = ? AND fi.romset = ? AND f.rowid = fi.data]]) - db.check("reading " .. tag1 .. " - " .. tag2 .. " - " .. set) - stmt:bind_values(tag1, tag2, set) - if stmt:step() == db.ROW then - data = stmt:get_value(0) + while not data do + local status = query:step() + if status == db.ROW then + data = query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check(string.format('reading %s data', file)) + break + end end - stmt:finalize() + query:reset() return data end return read @@ -22,49 +34,22 @@ function datfile.open(file, vertag, fixupcb) if not db then return nil end - local ver, dbver - local filepath - local fh - - for path in mame_manager.ui.options.entries.historypath:value():gmatch("([^;]+)") do - filepath = emu.subst_env(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end - -- remove unsafe chars from file for use in sql statement - file = file:gsub("[^%w%._]", "") - - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) - end - stmt:finalize() - if not fh and dbver then - -- data in database but missing file, just use what we have - return readret(file), dbver - elseif not fh then - return nil - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[_idx" ( - type VARCHAR NOT NULL, - val VARCHAR NOT NULL, - romset VARCHAR NOT NULL, - data INTEGER NOT NULL)]]) - db.check("creating index") - db.exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)") - db.check("creating table") - db.exec("CREATE INDEX \"typeval_" .. file .. "\" ON \"" .. file .. "_idx\"(type, val)") - db.check("creating typeval index") + local fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + return readret(file, tablename), dbver + else + return nil + end end + local ver if vertag then + -- scan file for version for line in fh:lines() do - local match = line:match(vertag .. "%s*(%S+)") + local match = line:match(vertag .. '%s*(%S+)') if match then ver = match break @@ -72,109 +57,204 @@ function datfile.open(file, vertag, fixupcb) end end if not ver then - -- use file ctime for version - ver = tostring(lfs.attributes(filepath, "change")) + -- fall back to file modification time for version + ver = tostring(lfs.attributes(filepath, 'change')) end if ver == dbver then fh:close() - return readret(file), dbver + return readret(file, tablename), dbver + end + + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s_idx" ( + type VARCHAR NOT NULL, + val VARCHAR NOT NULL, + romset VARCHAR NOT NULL, + data INTEGER NOT NULL);]], + tablename)) + db.check(string.format('creating %s index table', file)) + db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename)) + db.check(string.format('creating %s data table', file)) + db.exec( + string.format( + [[CREATE INDEX "typeval_%s" ON "%s_idx" (type, val, romset);]], + tablename, tablename)) + db.check(string.format('creating %s type/value index', file)) + end + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check(string.format('starting %s transaction', file)) then + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end end + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting") - db.exec("DELETE FROM \"" .. file .. "_idx\"") - db.check("deleting index") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end + db.exec(string.format([[DELETE FROM "%s_idx";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - - do - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning transaction") - local function gmatchpos() - local pos = 1 - local function iter() - local tags, data - while not data do - local npos - local spos, epos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", pos) - if not spos then - return nil - end - npos, epos = buffer:find("[\n\r]$%w+%s*[\n\r]+", epos) - if not npos then - return nil - end - tags = buffer:sub(spos, epos) - spos, npos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", epos) - if not spos then - return nil - end - data = buffer:sub(epos, spos) - pos = spos + db.set_version(file, ver) + if not db.check(string.format('updating %s version', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end + + local dataquery = db.prepare( + string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename)) + local indexquery = db.prepare( + string.format( + [[INSERT INTO "%s_idx" (type, val, romset, data) VALUES (?, ?, ?, ?)]], + tablename)) + + fh:seek('set') + local buffer = fh:read('a') + + local function gmatchpos() + local pos = 1 + local function iter() + local tags, data + while not data do + local npos + local spos, epos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', pos) + if not spos then + return nil + end + npos, epos = buffer:find('[\n\r]$%w+%s*[\n\r]+', epos) + if not npos then + return nil end - return tags, data + tags = buffer:sub(spos, epos) + spos, npos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', epos) + if not spos then + return nil + end + data = buffer:sub(epos, spos) + pos = spos end - return iter + return tags, data end - for info, data in gmatchpos() do - local tags = {} - local infotype - info = info:gsub(utf8.char(0xfeff), "") --remove boms - data = data:gsub(utf8.char(0xfeff), "") - for s in info:gmatch("[\n\r]$([^\n\r]*)") do - if s:find("=", 1, true) then - local m1, m2 = s:match("([^=]*)=(.*)") - for tag in m1:gmatch("[^,]+") do - for set in m2:gmatch("[^,]+") do - tags[#tags + 1] = { tag = tag, set = set } - end + return iter + end + + for info, data in gmatchpos() do + local tags = {} + local infotype + info = info:gsub(utf8.char(0xfeff), '') -- remove byte order marks + data = data:gsub(utf8.char(0xfeff), '') + for s in info:gmatch('[\n\r]$([^\n\r]*)') do + if s:find('=', 1, true) then + local m1, m2 = s:match('([^=]*)=(.*)') + for tag in m1:gmatch('[^,]+') do + for set in m2:gmatch('[^,]+') do + table.insert(tags, { tag = tag, set = set }) end - else - infotype = s - break end + else + infotype = s + break end + end + + data = data:gsub('[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].*', '') - data = data:gsub("[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]", "\n") - data = data:gsub("[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]", "\n") - data = data:gsub("[\n\r]$end%s*[\n\r].*", "") + if (#tags > 0) and infotype then + data = data:gsub('\r', '') -- strip carriage returns + if fixupcb then + data = fixupcb(data) + end - if #tags > 0 and infotype then - data = data:gsub("\r", "") -- strip crs - if fixupcb then - data = fixupcb(data) + dataquery:bind_values(data) + local row + while true do + local status = dataquery:step() + if status == db.DONE then + row = dataquery:last_insert_rowid(); + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break end - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?)") - db.check("inserting values") - stmt:bind_values(data) - stmt:step() - local row = stmt:last_insert_rowid() - stmt:finalize() + end + dataquery:reset() + + if row then for num, tag in pairs(tags) do - stmt = db.prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?, ?)") - db.check("inserting into index") - stmt:bind_values(infotype, tag.tag, tag.set, row) - stmt:step() - stmt:finalize() + indexquery:bind_values(infotype, tag.tag, tag.set, row) + while true do + local status = indexquery:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + indexquery:reset() end end end - db.exec("END TRANSACTION") - db.check("ending transaction") end + + dataquery:finalize() + indexquery:finalize() + fh:close() + db.exec([[COMMIT TRANSACTION;]]) + if not db.check(string.format('committing %s transaction', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end - return readret(file), ver + return readret(file, tablename), ver end return datfile |