summaryrefslogtreecommitdiffstats
path: root/plugins/data/load_dat.lua
diff options
context:
space:
mode:
author Vas Crabb <vas@vastheman.com>2022-06-25 04:57:24 +1000
committer Vas Crabb <vas@vastheman.com>2022-06-25 04:57:24 +1000
commit923ef2c25db75ccb6a6d747304fc9c79ef454a46 (patch)
tree1a53bc7898adc2a7015a5a9e208c5df8cb024023 /plugins/data/load_dat.lua
parent2f453da00cab66472bde7bc40ca22ce7f4b89e74 (diff)
-plugins.data: Reworked the code substantialy.
* Use the plugin data folder for storing the cache. The history folder may be read-only or shared with different configurations. * Don't create the cache database or surrounding folder if there's nothing to store in it. * Actually use prepared queries multiple times rather than always destroying them after a single use. * Added proper error checking for most database operations. * Improved query performance by avoiding outer joins and table scans. -bus/nubus: Made the Macintosh Display Cards map the blue channel to white with monochrome monitors. Also added logging for PLL configuration to help debug how CRTC and RAMDAC clocks work in the future.
Diffstat (limited to 'plugins/data/load_dat.lua')
-rw-r--r--plugins/data/load_dat.lua336
1 files changed, 208 insertions, 128 deletions
diff --git a/plugins/data/load_dat.lua b/plugins/data/load_dat.lua
index 7209872ad32..0438c5499ba 100644
--- a/plugins/data/load_dat.lua
+++ b/plugins/data/load_dat.lua
@@ -1,17 +1,29 @@
local datfile = {}
-local db = require("data/database")
-local function readret(file)
- local function read(tag1, tag2, set)
+local db = require('data/database')
+
+local function readret(file, tablename)
+ local query = db.prepare(
+ string.format(
+ [[SELECT f.data
+ FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid
+ WHERE fi.type = ? AND fi.val = ? AND fi.romset = ?;]],
+ tablename, tablename))
+ local function read(tag, val, set)
+ query:bind_values(tag, val, set)
local data
- local stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [["
- AS f WHERE fi.type = ? AND fi.val = ? AND fi.romset = ? AND f.rowid = fi.data]])
- db.check("reading " .. tag1 .. " - " .. tag2 .. " - " .. set)
- stmt:bind_values(tag1, tag2, set)
- if stmt:step() == db.ROW then
- data = stmt:get_value(0)
+ while not data do
+ local status = query:step()
+ if status == db.ROW then
+ data = query:get_value(0)
+ elseif status == db.DONE then
+ break
+ elseif status ~= db.BUSY then
+ db.check(string.format('reading %s data', file))
+ break
+ end
end
- stmt:finalize()
+ query:reset()
return data
end
return read
@@ -22,49 +34,22 @@ function datfile.open(file, vertag, fixupcb)
if not db then
return nil
end
- local ver, dbver
- local filepath
- local fh
-
- for path in mame_manager.ui.options.entries.historypath:value():gmatch("([^;]+)") do
- filepath = emu.subst_env(path) .. "/" .. file
- fh = io.open(filepath, "r")
- if fh then
- break
- end
- end
- -- remove unsafe chars from file for use in sql statement
- file = file:gsub("[^%w%._]", "")
-
- local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?")
- db.check("reading version")
- stmt:bind_values(file)
- if stmt:step() == db.ROW then
- dbver = stmt:get_value(0)
- end
- stmt:finalize()
- if not fh and dbver then
- -- data in database but missing file, just use what we have
- return readret(file), dbver
- elseif not fh then
- return nil
- elseif not dbver then
- db.exec("CREATE TABLE \"" .. file .. [[_idx" (
- type VARCHAR NOT NULL,
- val VARCHAR NOT NULL,
- romset VARCHAR NOT NULL,
- data INTEGER NOT NULL)]])
- db.check("creating index")
- db.exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)")
- db.check("creating table")
- db.exec("CREATE INDEX \"typeval_" .. file .. "\" ON \"" .. file .. "_idx\"(type, val)")
- db.check("creating typeval index")
+ local fh, filepath, tablename, dbver = db.open_data_file(file)
+ if not fh then
+ if dbver then
+ -- data in database but missing file, just use what we have
+ return readret(file, tablename), dbver
+ else
+ return nil
+ end
end
+ local ver
if vertag then
+ -- scan file for version
for line in fh:lines() do
- local match = line:match(vertag .. "%s*(%S+)")
+ local match = line:match(vertag .. '%s*(%S+)')
if match then
ver = match
break
@@ -72,109 +57,204 @@ function datfile.open(file, vertag, fixupcb)
end
end
if not ver then
- -- use file ctime for version
- ver = tostring(lfs.attributes(filepath, "change"))
+ -- fall back to file modification time for version
+ ver = tostring(lfs.attributes(filepath, 'change'))
end
if ver == dbver then
fh:close()
- return readret(file), dbver
+ return readret(file, tablename), dbver
+ end
+
+ if not dbver then
+ db.exec(
+ string.format(
+ [[CREATE TABLE "%s_idx" (
+ type VARCHAR NOT NULL,
+ val VARCHAR NOT NULL,
+ romset VARCHAR NOT NULL,
+ data INTEGER NOT NULL);]],
+ tablename))
+ db.check(string.format('creating %s index table', file))
+ db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename))
+ db.check(string.format('creating %s data table', file))
+ db.exec(
+ string.format(
+ [[CREATE INDEX "typeval_%s" ON "%s_idx" (type, val, romset);]],
+ tablename, tablename))
+ db.check(string.format('creating %s type/value index', file))
+ end
+
+ db.exec([[BEGIN TRANSACTION;]])
+ if not db.check(string.format('starting %s transaction', file)) then
+ fh:close()
+ if dbver then
+ return readret(file, tablename), dbver
+ else
+ return nil
+ end
end
+ -- clean out previous data and update the version
if dbver then
- db.exec("DELETE FROM \"" .. file .. "\"")
- db.check("deleting")
- db.exec("DELETE FROM \"" .. file .. "_idx\"")
- db.check("deleting index")
- stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?")
- db.check("updating version")
- else
- stmt = db.prepare("INSERT INTO version VALUES (?, ?)")
- db.check("inserting version")
+ db.exec(string.format([[DELETE FROM "%s";]], tablename))
+ if not db.check(string.format('deleting previous %s data', file)) then
+ db.exec([[ROLLBACK TRANSACTION;]])
+ fh:close()
+ return readret(file, tablename), dbver
+ end
+ db.exec(string.format([[DELETE FROM "%s_idx";]], tablename))
+ if not db.check(string.format('deleting previous %s data', file)) then
+ db.exec([[ROLLBACK TRANSACTION;]])
+ fh:close()
+ return readret(file, tablename), dbver
+ end
end
- stmt:bind_values(ver, file)
- stmt:step()
- stmt:finalize()
-
- do
- fh:seek("set")
- local buffer = fh:read("a")
- db.exec("BEGIN TRANSACTION")
- db.check("beginning transaction")
- local function gmatchpos()
- local pos = 1
- local function iter()
- local tags, data
- while not data do
- local npos
- local spos, epos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", pos)
- if not spos then
- return nil
- end
- npos, epos = buffer:find("[\n\r]$%w+%s*[\n\r]+", epos)
- if not npos then
- return nil
- end
- tags = buffer:sub(spos, epos)
- spos, npos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", epos)
- if not spos then
- return nil
- end
- data = buffer:sub(epos, spos)
- pos = spos
+ db.set_version(file, ver)
+ if not db.check(string.format('updating %s version', file)) then
+ db.exec([[ROLLBACK TRANSACTION;]])
+ fh:close()
+ if dbver then
+ return readret(file, tablename), dbver
+ else
+ return nil
+ end
+ end
+
+ local dataquery = db.prepare(
+ string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename))
+ local indexquery = db.prepare(
+ string.format(
+ [[INSERT INTO "%s_idx" (type, val, romset, data) VALUES (?, ?, ?, ?)]],
+ tablename))
+
+ fh:seek('set')
+ local buffer = fh:read('a')
+
+ local function gmatchpos()
+ local pos = 1
+ local function iter()
+ local tags, data
+ while not data do
+ local npos
+ local spos, epos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', pos)
+ if not spos then
+ return nil
+ end
+ npos, epos = buffer:find('[\n\r]$%w+%s*[\n\r]+', epos)
+ if not npos then
+ return nil
end
- return tags, data
+ tags = buffer:sub(spos, epos)
+ spos, npos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', epos)
+ if not spos then
+ return nil
+ end
+ data = buffer:sub(epos, spos)
+ pos = spos
end
- return iter
+ return tags, data
end
- for info, data in gmatchpos() do
- local tags = {}
- local infotype
- info = info:gsub(utf8.char(0xfeff), "") --remove boms
- data = data:gsub(utf8.char(0xfeff), "")
- for s in info:gmatch("[\n\r]$([^\n\r]*)") do
- if s:find("=", 1, true) then
- local m1, m2 = s:match("([^=]*)=(.*)")
- for tag in m1:gmatch("[^,]+") do
- for set in m2:gmatch("[^,]+") do
- tags[#tags + 1] = { tag = tag, set = set }
- end
+ return iter
+ end
+
+ for info, data in gmatchpos() do
+ local tags = {}
+ local infotype
+ info = info:gsub(utf8.char(0xfeff), '') -- remove byte order marks
+ data = data:gsub(utf8.char(0xfeff), '')
+ for s in info:gmatch('[\n\r]$([^\n\r]*)') do
+ if s:find('=', 1, true) then
+ local m1, m2 = s:match('([^=]*)=(.*)')
+ for tag in m1:gmatch('[^,]+') do
+ for set in m2:gmatch('[^,]+') do
+ table.insert(tags, { tag = tag, set = set })
end
- else
- infotype = s
- break
end
+ else
+ infotype = s
+ break
end
+ end
+
+ data = data:gsub('[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]', '\n')
+ data = data:gsub('[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]', '\n')
+ data = data:gsub('[\n\r]$end%s*[\n\r].*', '')
- data = data:gsub("[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]", "\n")
- data = data:gsub("[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]", "\n")
- data = data:gsub("[\n\r]$end%s*[\n\r].*", "")
+ if (#tags > 0) and infotype then
+ data = data:gsub('\r', '') -- strip carriage returns
+ if fixupcb then
+ data = fixupcb(data)
+ end
- if #tags > 0 and infotype then
- data = data:gsub("\r", "") -- strip crs
- if fixupcb then
- data = fixupcb(data)
+ dataquery:bind_values(data)
+ local row
+ while true do
+ local status = dataquery:step()
+ if status == db.DONE then
+ row = dataquery:last_insert_rowid();
+ break
+ elseif status == db.BUSY then
+ emu.print_error(string.format('Database busy: inserting %s data', file))
+ dataquery:finalize()
+ indexquery:finalize()
+ db.exec([[ROLLBACK TRANSACTION;]])
+ fh:close()
+ if dbver then
+ return readret(file, tablename), dbver
+ else
+ return nil
+ end
+ elseif result ~= db.ROW then
+ db.check(string.format('inserting %s data', file))
+ break
end
- stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?)")
- db.check("inserting values")
- stmt:bind_values(data)
- stmt:step()
- local row = stmt:last_insert_rowid()
- stmt:finalize()
+ end
+ dataquery:reset()
+
+ if row then
for num, tag in pairs(tags) do
- stmt = db.prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?, ?)")
- db.check("inserting into index")
- stmt:bind_values(infotype, tag.tag, tag.set, row)
- stmt:step()
- stmt:finalize()
+ indexquery:bind_values(infotype, tag.tag, tag.set, row)
+ while true do
+ local status = indexquery:step()
+ if status == db.DONE then
+ break
+ elseif status == db.BUSY then
+ emu.print_error(string.format('Database busy: inserting %s data', file))
+ dataquery:finalize()
+ indexquery:finalize()
+ db.exec([[ROLLBACK TRANSACTION;]])
+ fh:close()
+ if dbver then
+ return readret(file, tablename), dbver
+ else
+ return nil
+ end
+ elseif result ~= db.ROW then
+ db.check(string.format('inserting %s data', file))
+ break
+ end
+ end
+ indexquery:reset()
end
end
end
- db.exec("END TRANSACTION")
- db.check("ending transaction")
end
+
+ dataquery:finalize()
+ indexquery:finalize()
+
fh:close()
+ db.exec([[COMMIT TRANSACTION;]])
+ if not db.check(string.format('committing %s transaction', file)) then
+ db.exec([[ROLLBACK TRANSACTION;]])
+ if dbver then
+ return readret(file, tablename), dbver
+ else
+ return nil
+ end
+ end
- return readret(file), ver
+ return readret(file, tablename), ver
end
return datfile