diff options
Diffstat (limited to 'plugins/data/load_dat.lua')
-rw-r--r-- | plugins/data/load_dat.lua | 336 |
1 files changed, 208 insertions, 128 deletions
diff --git a/plugins/data/load_dat.lua b/plugins/data/load_dat.lua index 7209872ad32..0438c5499ba 100644 --- a/plugins/data/load_dat.lua +++ b/plugins/data/load_dat.lua @@ -1,17 +1,29 @@ local datfile = {} -local db = require("data/database") -local function readret(file) - local function read(tag1, tag2, set) +local db = require('data/database') + +local function readret(file, tablename) + local query = db.prepare( + string.format( + [[SELECT f.data + FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid + WHERE fi.type = ? AND fi.val = ? AND fi.romset = ?;]], + tablename, tablename)) + local function read(tag, val, set) + query:bind_values(tag, val, set) local data - local stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [[" - AS f WHERE fi.type = ? AND fi.val = ? AND fi.romset = ? AND f.rowid = fi.data]]) - db.check("reading " .. tag1 .. " - " .. tag2 .. " - " .. set) - stmt:bind_values(tag1, tag2, set) - if stmt:step() == db.ROW then - data = stmt:get_value(0) + while not data do + local status = query:step() + if status == db.ROW then + data = query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check(string.format('reading %s data', file)) + break + end end - stmt:finalize() + query:reset() return data end return read @@ -22,49 +34,22 @@ function datfile.open(file, vertag, fixupcb) if not db then return nil end - local ver, dbver - local filepath - local fh - - for path in mame_manager.ui.options.entries.historypath:value():gmatch("([^;]+)") do - filepath = emu.subst_env(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end - -- remove unsafe chars from file for use in sql statement - file = file:gsub("[^%w%._]", "") - - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) - end - stmt:finalize() - if not fh and dbver then - -- data in database but missing file, just use what we have - return readret(file), dbver - elseif not fh then - return nil - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[_idx" ( - type VARCHAR NOT NULL, - val VARCHAR NOT NULL, - romset VARCHAR NOT NULL, - data INTEGER NOT NULL)]]) - db.check("creating index") - db.exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)") - db.check("creating table") - db.exec("CREATE INDEX \"typeval_" .. file .. "\" ON \"" .. file .. "_idx\"(type, val)") - db.check("creating typeval index") + local fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + return readret(file, tablename), dbver + else + return nil + end end + local ver if vertag then + -- scan file for version for line in fh:lines() do - local match = line:match(vertag .. "%s*(%S+)") + local match = line:match(vertag .. '%s*(%S+)') if match then ver = match break @@ -72,109 +57,204 @@ function datfile.open(file, vertag, fixupcb) end end if not ver then - -- use file ctime for version - ver = tostring(lfs.attributes(filepath, "change")) + -- fall back to file modification time for version + ver = tostring(lfs.attributes(filepath, 'change')) end if ver == dbver then fh:close() - return readret(file), dbver + return readret(file, tablename), dbver + end + + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s_idx" ( + type VARCHAR NOT NULL, + val VARCHAR NOT NULL, + romset VARCHAR NOT NULL, + data INTEGER NOT NULL);]], + tablename)) + db.check(string.format('creating %s index table', file)) + db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename)) + db.check(string.format('creating %s data table', file)) + db.exec( + string.format( + [[CREATE INDEX "typeval_%s" ON "%s_idx" (type, val, romset);]], + tablename, tablename)) + db.check(string.format('creating %s type/value index', file)) + end + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check(string.format('starting %s transaction', file)) then + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end end + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting") - db.exec("DELETE FROM \"" .. file .. "_idx\"") - db.check("deleting index") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end + db.exec(string.format([[DELETE FROM "%s_idx";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - - do - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning transaction") - local function gmatchpos() - local pos = 1 - local function iter() - local tags, data - while not data do - local npos - local spos, epos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", pos) - if not spos then - return nil - end - npos, epos = buffer:find("[\n\r]$%w+%s*[\n\r]+", epos) - if not npos then - return nil - end - tags = buffer:sub(spos, epos) - spos, npos = buffer:find("[\n\r]$[^=\n\r]*=[^\n\r]*", epos) - if not spos then - return nil - end - data = buffer:sub(epos, spos) - pos = spos + db.set_version(file, ver) + if not db.check(string.format('updating %s version', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end + + local dataquery = db.prepare( + string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename)) + local indexquery = db.prepare( + string.format( + [[INSERT INTO "%s_idx" (type, val, romset, data) VALUES (?, ?, ?, ?)]], + tablename)) + + fh:seek('set') + local buffer = fh:read('a') + + local function gmatchpos() + local pos = 1 + local function iter() + local tags, data + while not data do + local npos + local spos, epos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', pos) + if not spos then + return nil + end + npos, epos = buffer:find('[\n\r]$%w+%s*[\n\r]+', epos) + if not npos then + return nil end - return tags, data + tags = buffer:sub(spos, epos) + spos, npos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', epos) + if not spos then + return nil + end + data = buffer:sub(epos, spos) + pos = spos end - return iter + return tags, data end - for info, data in gmatchpos() do - local tags = {} - local infotype - info = info:gsub(utf8.char(0xfeff), "") --remove boms - data = data:gsub(utf8.char(0xfeff), "") - for s in info:gmatch("[\n\r]$([^\n\r]*)") do - if s:find("=", 1, true) then - local m1, m2 = s:match("([^=]*)=(.*)") - for tag in m1:gmatch("[^,]+") do - for set in m2:gmatch("[^,]+") do - tags[#tags + 1] = { tag = tag, set = set } - end + return iter + end + + for info, data in gmatchpos() do + local tags = {} + local infotype + info = info:gsub(utf8.char(0xfeff), '') -- remove byte order marks + data = data:gsub(utf8.char(0xfeff), '') + for s in info:gmatch('[\n\r]$([^\n\r]*)') do + if s:find('=', 1, true) then + local m1, m2 = s:match('([^=]*)=(.*)') + for tag in m1:gmatch('[^,]+') do + for set in m2:gmatch('[^,]+') do + table.insert(tags, { tag = tag, set = set }) end - else - infotype = s - break end + else + infotype = s + break end + end + + data = data:gsub('[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].*', '') - data = data:gsub("[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]", "\n") - data = data:gsub("[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]", "\n") - data = data:gsub("[\n\r]$end%s*[\n\r].*", "") + if (#tags > 0) and infotype then + data = data:gsub('\r', '') -- strip carriage returns + if fixupcb then + data = fixupcb(data) + end - if #tags > 0 and infotype then - data = data:gsub("\r", "") -- strip crs - if fixupcb then - data = fixupcb(data) + dataquery:bind_values(data) + local row + while true do + local status = dataquery:step() + if status == db.DONE then + row = dataquery:last_insert_rowid(); + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break end - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?)") - db.check("inserting values") - stmt:bind_values(data) - stmt:step() - local row = stmt:last_insert_rowid() - stmt:finalize() + end + dataquery:reset() + + if row then for num, tag in pairs(tags) do - stmt = db.prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?, ?)") - db.check("inserting into index") - stmt:bind_values(infotype, tag.tag, tag.set, row) - stmt:step() - stmt:finalize() + indexquery:bind_values(infotype, tag.tag, tag.set, row) + while true do + local status = indexquery:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + indexquery:reset() end end end - db.exec("END TRANSACTION") - db.check("ending transaction") end + + dataquery:finalize() + indexquery:finalize() + fh:close() + db.exec([[COMMIT TRANSACTION;]]) + if not db.check(string.format('committing %s transaction', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end - return readret(file), ver + return readret(file, tablename), ver end return datfile |