diff options
Diffstat (limited to 'plugins/data')
-rw-r--r-- | plugins/data/button_char.lua | 6 | ||||
-rw-r--r-- | plugins/data/data_command.lua | 18 | ||||
-rw-r--r-- | plugins/data/data_gameinit.lua | 10 | ||||
-rw-r--r-- | plugins/data/data_hiscore.lua | 674 | ||||
-rw-r--r-- | plugins/data/data_history.lua | 208 | ||||
-rw-r--r-- | plugins/data/data_mameinfo.lua | 20 | ||||
-rw-r--r-- | plugins/data/data_marp.lua | 186 | ||||
-rw-r--r-- | plugins/data/data_messinfo.lua | 19 | ||||
-rw-r--r-- | plugins/data/data_story.lua | 20 | ||||
-rw-r--r-- | plugins/data/data_sysinfo.lua | 9 | ||||
-rw-r--r-- | plugins/data/database.lua | 170 | ||||
-rw-r--r-- | plugins/data/init.lua | 126 | ||||
-rw-r--r-- | plugins/data/load_dat.lua | 339 |
13 files changed, 1355 insertions, 450 deletions
diff --git a/plugins/data/button_char.lua b/plugins/data/button_char.lua index cee756c2603..02c64d9673c 100644 --- a/plugins/data/button_char.lua +++ b/plugins/data/button_char.lua @@ -236,9 +236,9 @@ local convert_text = } local function convert_char(str) - str = str:gsub("@([a-zA-Z%-]+)", function(s) if convert_text[s] then return utf8.char(convert_text[s] + 0xe000) end return s end) - str = str:gsub("_(.)", function(s) if default_text[s] then return utf8.char(default_text[s] + 0xe000) end return s end) - str = str:gsub("%^(.)", function(s) if expand_text[s] then return utf8.char(expand_text[s] + 0xe000) end return s end) + str = str:gsub("@(%g+)", function(s) if convert_text[s] then return utf8.char(convert_text[s] + 0xe000) end return s end) + str = str:gsub("_(%g)", function(s) if default_text[s] then return utf8.char(default_text[s] + 0xe000) end return s end) + str = str:gsub("%^(%g)", function(s) if expand_text[s] then return utf8.char(expand_text[s] + 0xe000) end return s end) return str end diff --git a/plugins/data/data_command.lua b/plugins/data/data_command.lua index e85533d152c..910a20beadc 100644 --- a/plugins/data/data_command.lua +++ b/plugins/data/data_command.lua @@ -1,9 +1,16 @@ local dat = {} + local info, ver -local datread = require("data/load_dat") +local datread = require('data/load_dat') do - local convert = require("data/button_char") - datread, ver = datread.open("command.dat", "# Command List%-.+hand", convert) + local buttonchar + local function convert(str) + if not buttonchar then + buttonchar = require("data/button_char") + end + return buttonchar(str) + end + datread, ver = datread.open('command.dat', '#[^V]*Ver[^.:]*[.:]', convert) end function dat.check(set, softlist) @@ -11,12 +18,11 @@ function dat.check(set, softlist) return nil end local status - status, info = pcall(datread, "cmd", "info", set) + status, info = pcall(datread, 'cmd', 'info', set) if not status or not info then return nil end - info = "#jf\n" .. info - return _("Command") + return _p('plugin-data', 'Command') end function dat.get() diff --git a/plugins/data/data_gameinit.lua b/plugins/data/data_gameinit.lua index 7b93334e2ce..f6856629bb8 100644 --- a/plugins/data/data_gameinit.lua +++ b/plugins/data/data_gameinit.lua @@ -1,19 +1,19 @@ local dat = {} -local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("gameinit.dat", "# .-GAMEINIT.DAT") +local ver, info +local datread = require('data/load_dat') +datread, ver = datread.open('gameinit.dat', '# .-GAMEINIT.DAT') function dat.check(set, softlist) if softlist or not datread then return nil end local status - status, info = pcall(datread, "mame", "info", set) + status, info = pcall(datread, 'mame', 'info', set) if not status or not info then return nil end - return _("Gameinit") + return _p('plugin-data', 'Gameinit') end function dat.get() diff --git a/plugins/data/data_hiscore.lua b/plugins/data/data_hiscore.lua index 9c888ddffdf..f05e4ddd173 100644 --- a/plugins/data/data_hiscore.lua +++ b/plugins/data/data_hiscore.lua @@ -1,15 +1,361 @@ --- to use this get the package from http://greatstone.free.fr/hi2txt/ +-- to use this get the package from https://greatstoneex.github.io/hi2txt-doc/ -- extract the hi2txt.zip and place it in your history path local dat = {} local env = {} local output local curset +local path = mame_manager.ui.options.entries.historypath:value():gsub("([^;]+)", "%1/hi2txt") +local htmlentites = { + ["amp"] = "&", + ["quot"] = utf8.char(34), + ["big-mid-dot"] = utf8.char(149), + ["copyright"] = utf8.char(169), + ["mid-dot"] = utf8.char(183), + ["one-on-two"] = utf8.char(189), + ["ring"] = utf8.char(214), + ["acute"] = utf8.char(225), + ["y-strike"] = utf8.char(590), + ["bridge1"] = utf8.char(761), + ["bridge2"] = utf8.char(765), + ["bridge3"] = utf8.char(766), + ["alpha"] = utf8.char(945), + ["beta"] = utf8.char(946), + ["gamma"] = utf8.char(947), + ["delta"] = utf8.char(948), + ["epsilon"] = utf8.char(949), + ["zeta"] = utf8.char(950), + ["eta"] = utf8.char(951), + ["theta"] = utf8.char(952), + ["iota"] = utf8.char(953), + ["kappa"] = utf8.char(954), + ["lambda"] = utf8.char(955), + ["mu"] = utf8.char(956), + ["nu"] = utf8.char(957), + ["xi"] = utf8.char(958), + ["omicron"] = utf8.char(959), + ["pi"] = utf8.char(960), + ["rho"] = utf8.char(961), + ["sigmaf"] = utf8.char(962), + ["sigma"] = utf8.char(963), + ["tau"] = utf8.char(964), + ["upsilon"] = utf8.char(965), + ["phi"] = utf8.char(966), + ["chi"] = utf8.char(967), + ["psi"] = utf8.char(968), + ["omega"] = utf8.char(969), + ["circle-line"] = utf8.char(984), + ["two-dots"] = utf8.char(1417), + ["inverted-question"] = utf8.char(1567), + ["rdquo"] = utf8.char(8221), + ["big-dot"] = utf8.char(8226), + ["three-dots"] = utf8.char(8230), + ["two-exclamations"] = utf8.char(8252), + ["broken-question"] = utf8.char(8253), + ["asterism"] = utf8.char(8258), + ["w-double-strike"] = utf8.char(8361), + ["square-2"] = utf8.char(8414), + ["roman-numeral-1"] = utf8.char(8544), + ["roman-numeral-2"] = utf8.char(8545), + ["roman-numeral-3"] = utf8.char(8546), + ["roman-numeral-4"] = utf8.char(8547), + ["roman-numeral-5"] = utf8.char(8548), + ["roman-numeral-6"] = utf8.char(8549), + ["roman-numeral-7"] = utf8.char(8550), + ["roman-numeral-8"] = utf8.char(8551), + ["roman-numeral-9"] = utf8.char(8552), + ["roman-numeral-10"] = utf8.char(8553), + ["roman-numeral-11"] = utf8.char(8554), + ["roman-numeral-12"] = utf8.char(8555), + ["small-roman-numeral-1"] = utf8.char(8560), + ["small-roman-numeral-2"] = utf8.char(8561), + ["small-roman-numeral-3"] = utf8.char(8562), + ["small-roman-numeral-4"] = utf8.char(8563), + ["small-roman-numeral-5"] = utf8.char(8564), + ["small-roman-numeral-6"] = utf8.char(8565), + ["small-roman-numeral-7"] = utf8.char(8566), + ["small-roman-numeral-8"] = utf8.char(8567), + ["small-roman-numeral-9"] = utf8.char(8568), + ["small-roman-numeral-10"] = utf8.char(8569), + ["small-roman-numeral-11"] = utf8.char(8570), + ["small-roman-numeral-12"] = utf8.char(8571), + ["left-arrow"] = utf8.char(8592), + ["right-double-arrow"] = utf8.char(8658), + ["four-lines"] = utf8.char(8803), + ["three-mid-dots"] = utf8.char(8943), + ["left-foot"] = utf8.char(8968), + ["right-foot"] = utf8.char(8968), + ["round-7"] = utf8.char(9318), + ["square"] = utf8.char(9633), + ["dot-in-square"] = utf8.char(9635), + ["h-lines-in-square"] = utf8.char(9636), + ["v-lines-in-square"] = utf8.char(9637), + ["slash-in-square"] = utf8.char(9639), + ["antislash-in-square"] = utf8.char(9640), + ["black-triangle-right"] = utf8.char(9658), + ["black-triangle-down"] = utf8.char(9660), + ["two-cubes"] = utf8.char(9707) , + ["umbrella"] = utf8.char(9730), + ["snowman"] = utf8.char(9731) , + ["black-star"] = utf8.char(9733), + ["star"] = utf8.char(9734), + ["headset"] = utf8.char(9738), + ["phone"] = utf8.char(9742), + ["hot-beverage"] = utf8.char(9749), + ["skull"] = utf8.char(9760), + ["ankh"] = utf8.char(9765), + ["cross-of-lorraine"] = utf8.char(9768), + ["cross-of-jerusalem"] = utf8.char(9769), + ["peace"] = utf8.char(9774), + ["angry-face"] = utf8.char(9785), + ["smiley"] = utf8.char(9786), + ["black-smiley"] = utf8.char(9787), + ["sun"] = utf8.char(9788), + ["moon"] = utf8.char(9789), + ["crescent-moon"] = utf8.char(9790), + ["woman"] = utf8.char(9792), + ["man"] = utf8.char(9794), + ["spaceship"] = utf8.char(9798), + ["aries"] = utf8.char(9800), + ["taurus"] = utf8.char(9801), + ["gemini"] = utf8.char(9802), + ["cancer"] = utf8.char(9803), + ["leo"] = utf8.char(9804), + ["virgo"] = utf8.char(9805), + ["libra"] = utf8.char(9806), + ["scorpio"] = utf8.char(9807), + ["sagitarius"] = utf8.char(9808), + ["capricorn"] = utf8.char(9809), + ["aquarius"] = utf8.char(9810), + ["pisces"] = utf8.char(9811), + ["amber"] = utf8.char(9816) , + ["black-spade"] = utf8.char(9824) , + ["heart"] = utf8.char(9825) , + ["black-club"] = utf8.char(9827) , + ["black-heart"] = utf8.char(9829) , + ["black-diamond"] = utf8.char(9830) , + ["single-music-note"] = utf8.char(9834), + ["double-music-note"] = utf8.char(9835), + ["crossed-swords"] = utf8.char(9876), + ["baseball"] = utf8.char(9918), + ["boat"] = utf8.char(9973), + ["scissors"] = utf8.char(9988), + ["airplane"] = utf8.char(9992), + ["multiplication"] = utf8.char(10005), + ["big-exclamation"] = utf8.char(10082), + ["left-black-heart"] = utf8.char(10085), + ["black-right-arrow-large"] = utf8.char(10152), + ["up-arrow-with-stroke"] = utf8.char(10505), + ["jp-h-a-small"] = utf8.char(12353), + ["jp-h-a"] = utf8.char(12354), + ["jp-h-i-small"] = utf8.char(12355), + ["jp-h-i"] = utf8.char(12356), + ["jp-h-u-small"] = utf8.char(12357), + ["jp-h-u"] = utf8.char(12358), + ["jp-h-e-small"] = utf8.char(12359), + ["jp-h-e"] = utf8.char(12360), + ["jp-h-o-small"] = utf8.char(12361), + ["jp-h-o"] = utf8.char(12362), + ["jp-h-ka"] = utf8.char(12363), + ["jp-h-ga"] = utf8.char(12364), + ["jp-h-ki"] = utf8.char(12365), + ["jp-h-gi"] = utf8.char(12366), + ["jp-h-ku"] = utf8.char(12367), + ["jp-h-gu"] = utf8.char(12368), + ["jp-h-ke"] = utf8.char(12369), + ["jp-h-ge"] = utf8.char(12370), + ["jp-h-ko"] = utf8.char(12371), + ["jp-h-go"] = utf8.char(12372), + ["jp-h-sa"] = utf8.char(12373), + ["jp-h-za"] = utf8.char(12374), + ["jp-h-si"] = utf8.char(12375), + ["jp-h-zi"] = utf8.char(12376), + ["jp-h-su"] = utf8.char(12377), + ["jp-h-zu"] = utf8.char(12378), + ["jp-h-se"] = utf8.char(12379), + ["jp-h-ze"] = utf8.char(12380), + ["jp-h-so"] = utf8.char(12381), + ["jp-h-zo"] = utf8.char(12382), + ["jp-h-ta"] = utf8.char(12383), + ["jp-h-da"] = utf8.char(12384), + ["jp-h-ti"] = utf8.char(12385), + ["jp-h-di"] = utf8.char(12386), + ["jp-h-tu-small"] = utf8.char(12387), + ["jp-h-sokuon"] = utf8.char(12387), + ["jp-h-tu"] = utf8.char(12388), + ["jp-h-du"] = utf8.char(12389), + ["jp-h-te"] = utf8.char(12390), + ["jp-h-de"] = utf8.char(12391), + ["jp-h-to"] = utf8.char(12392), + ["jp-h-do"] = utf8.char(12393), + ["jp-h-na"] = utf8.char(12394), + ["jp-h-ni"] = utf8.char(12395), + ["jp-h-nu"] = utf8.char(12396), + ["jp-h-ne"] = utf8.char(12397), + ["jp-h-no"] = utf8.char(12398), + ["jp-h-ha"] = utf8.char(12399), + ["jp-h-ba"] = utf8.char(12400), + ["jp-h-pa"] = utf8.char(12401), + ["jp-h-hi"] = utf8.char(12402), + ["jp-h-bi"] = utf8.char(12403), + ["jp-h-pi"] = utf8.char(12404), + ["jp-h-hu"] = utf8.char(12405), + ["jp-h-bu"] = utf8.char(12406), + ["jp-h-pu"] = utf8.char(12407), + ["jp-h-he"] = utf8.char(12408), + ["jp-h-be"] = utf8.char(12409), + ["jp-h-pe"] = utf8.char(12410), + ["jp-h-ho"] = utf8.char(12411), + ["jp-h-bo"] = utf8.char(12412), + ["jp-h-po"] = utf8.char(12413), + ["jp-h-ma"] = utf8.char(12414), + ["jp-h-mi"] = utf8.char(12415), + ["jp-h-mu"] = utf8.char(12416), + ["jp-h-me"] = utf8.char(12417), + ["jp-h-mo"] = utf8.char(12418), + ["jp-h-ya-small"] = utf8.char(12419), + ["jp-h-youon-a"] = utf8.char(12419), + ["jp-h-ya"] = utf8.char(12420), + ["jp-h-yu-small"] = utf8.char(12421), + ["jp-h-youon-u"] = utf8.char(12421), + ["jp-h-yu"] = utf8.char(12422), + ["jp-h-yo-small"] = utf8.char(12423), + ["jp-h-youon-o"] = utf8.char(12423), + ["jp-h-yo"] = utf8.char(12424), + ["jp-h-ra"] = utf8.char(12425), + ["jp-h-ri"] = utf8.char(12426), + ["jp-h-ru"] = utf8.char(12427), + ["jp-h-re"] = utf8.char(12428), + ["jp-h-ro"] = utf8.char(12429), + ["jp-h-wa-small"] = utf8.char(12430), + ["jp-h-wa"] = utf8.char(12431), + ["jp-h-wi"] = utf8.char(12432), + ["jp-h-we"] = utf8.char(12433), + ["jp-h-wo"] = utf8.char(12434), + ["jp-h-n"] = utf8.char(12435), + ["jp-h-vu"] = utf8.char(12436), + ["jp-h-ka-small"] = utf8.char(12437), + ["jp-h-ke-small"] = utf8.char(12438), + ["jp-h-dakuten"] = utf8.char(12443), + ["jp-h-handakuten"] = utf8.char(12444), + ["jp-k-a-small"] = utf8.char(12449), + ["jp-k-a"] = utf8.char(12450), + ["jp-k-i-small"] = utf8.char(12451), + ["jp-k-i"] = utf8.char(12452), + ["jp-k-u-small"] = utf8.char(12453), + ["jp-k-u"] = utf8.char(12454), + ["jp-k-e-small"] = utf8.char(12455), + ["jp-k-e"] = utf8.char(12456), + ["jp-k-o-small"] = utf8.char(12457), + ["jp-k-o"] = utf8.char(12458), + ["jp-k-ka"] = utf8.char(12459), + ["jp-k-ga"] = utf8.char(12460), + ["jp-k-ki"] = utf8.char(12461), + ["jp-k-gi"] = utf8.char(12462), + ["jp-k-ku"] = utf8.char(12463), + ["jp-k-gu"] = utf8.char(12464), + ["jp-k-ke"] = utf8.char(12465), + ["jp-k-ge"] = utf8.char(12466), + ["jp-k-ko"] = utf8.char(12467), + ["jp-k-go"] = utf8.char(12468), + ["jp-k-sa"] = utf8.char(12469), + ["jp-k-za"] = utf8.char(12470), + ["jp-k-si"] = utf8.char(12471), + ["jp-k-zi"] = utf8.char(12472), + ["jp-k-su"] = utf8.char(12473), + ["jp-k-zu"] = utf8.char(12474), + ["jp-k-se"] = utf8.char(12475), + ["jp-k-ze"] = utf8.char(12476), + ["jp-k-so"] = utf8.char(12477), + ["jp-k-zo"] = utf8.char(12478), + ["jp-k-ta"] = utf8.char(12479), + ["jp-k-da"] = utf8.char(12480), + ["jp-k-ti"] = utf8.char(12481), + ["jp-k-di"] = utf8.char(12482), + ["jp-k-tu-small"] = utf8.char(12483), + ["jp-k-tu"] = utf8.char(12484), + ["jp-k-du"] = utf8.char(12485), + ["jp-k-te"] = utf8.char(12486), + ["jp-k-de"] = utf8.char(12487), + ["jp-k-to"] = utf8.char(12488), + ["jp-k-do"] = utf8.char(12489), + ["jp-k-na"] = utf8.char(12490), + ["jp-k-ni"] = utf8.char(12491), + ["jp-k-nu"] = utf8.char(12492), + ["jp-k-ne"] = utf8.char(12493), + ["jp-k-no"] = utf8.char(12494), + ["jp-k-ha"] = utf8.char(12495), + ["jp-k-ba"] = utf8.char(12496), + ["jp-k-pa"] = utf8.char(12497), + ["jp-k-hi"] = utf8.char(12498), + ["jp-k-bi"] = utf8.char(12499), + ["jp-k-pi"] = utf8.char(12500), + ["jp-k-hu"] = utf8.char(12501), + ["jp-k-bu"] = utf8.char(12502), + ["jp-k-pu"] = utf8.char(12503), + ["jp-k-he"] = utf8.char(12504), + ["jp-k-be"] = utf8.char(12505), + ["jp-k-pe"] = utf8.char(12506), + ["jp-k-ho"] = utf8.char(12507), + ["jp-k-bo"] = utf8.char(12508), + ["jp-k-po"] = utf8.char(12509), + ["jp-k-ma"] = utf8.char(12510), + ["jp-k-mi"] = utf8.char(12511), + ["jp-k-mu"] = utf8.char(12512), + ["jp-k-me"] = utf8.char(12513), + ["jp-k-mo"] = utf8.char(12514), + ["jp-k-ya-small"] = utf8.char(12515), + ["jp-k-ya"] = utf8.char(12516), + ["jp-k-yu-small"] = utf8.char(12517), + ["jp-k-yu"] = utf8.char(12518), + ["jp-k-yo-small"] = utf8.char(12519), + ["jp-k-yo"] = utf8.char(12520), + ["jp-k-ra"] = utf8.char(12521), + ["jp-k-ri"] = utf8.char(12522), + ["jp-k-ru"] = utf8.char(12523), + ["jp-k-re"] = utf8.char(12524), + ["jp-k-ro"] = utf8.char(12525), + ["jp-k-wa-small"] = utf8.char(12526), + ["jp-k-wa"] = utf8.char(12527), + ["jp-k-wi"] = utf8.char(12528), + ["jp-k-we"] = utf8.char(12529), + ["jp-k-wo"] = utf8.char(12530), + ["jp-k-n"] = utf8.char(12531), + ["jp-k-vu"] = utf8.char(12532), + ["jp-k-ka-small"] = utf8.char(12533), + ["jp-k-ke-small"] = utf8.char(12534), + ["jp-k-va"] = utf8.char(12535), + ["jp-k-vi"] = utf8.char(12536), + ["jp-k-ve"] = utf8.char(12537), + ["jp-k-vo"] = utf8.char(12538), + ["jp-k-zero"] = utf8.char(38646), + ["jp-k-one"] = utf8.char(19968), + ["jp-k-two"] = utf8.char(20108), + ["jp-k-three"] = utf8.char(19977), + ["jp-k-four"] = utf8.char(22235), + ["jp-k-five"] = utf8.char(20116), + ["jp-k-six"] = utf8.char(20845), + ["jp-k-seven"] = utf8.char(19971), + ["jp-k-height"] = utf8.char(20843), + ["jp-k-nine"] = utf8.char(20061), + ["lama"] = "lama", + ["cat-face"] = utf8.char(9786) , + ["whale"] = "whale", + ["thumbs-up"] = utf8.char(8730) , + ["shoe"] = "shoe", + ["kiss"] = utf8.char(9786), + ["heart-with-arrow"] = utf8.char(9829), + ["car-side"] = utf8.char(9936), + ["car-front"] = utf8.char(9936), + ["mens-symbol"] = utf8.char(9794), + ["womens-symbol"] = utf8.char(9792), + ["feet"] = utf8.char(128062) +} -function env.open(file, size) +function env.open(file, size, swap) if file == ".hi" then local path = "hi" - local ini = emu.file(lfs.env_replace(manager:options().entries.inipath:value()), 1) + local ini = emu.file(manager.options.entries.inipath:value(), 1) local ret = ini:open("hiscore.ini") if not ret then local inifile = ini:read(ini:size()) @@ -23,7 +369,7 @@ function env.open(file, size) end file = path .. "/" .. curset .. ".hi" else - file = lfs.env_replace(manager:options().entries.nvram_directory:value()) .. "/" .. curset .. "/" .. file + file = manager.options.entries.nvram_directory:value() .. "/" .. curset .. "/" .. file end local f = io.open(file, "rb") local content = f:read("*all") @@ -31,6 +377,16 @@ function env.open(file, size) if #content < size then content = content .. string.rep("\0", size - #content) end + if swap then + if swap == 2 then + content = content:gsub("(.)(.)", function(c1, c2) return c2 .. c1 end) + elseif swap == 4 then + content = content:gsub("(.)(.)(.)(.)", function(c1, c2, c3, c4) return c4 .. c3 .. c2 .. c1 end) + else + emu.print_verbose("swap " .. swap .. " not supported") + end + end + return content end @@ -86,8 +442,9 @@ end function env.byte_trim(bytes, val) val = tonumber(val) - for i = 1, #bytes do - if bytes[i] ~= val then + len = #bytes + for i = 1, len do + if bytes[1] ~= val then return bytes end table.remove(bytes, 1) @@ -95,6 +452,19 @@ function env.byte_trim(bytes, val) return bytes end +function env.byte_trunc(bytes, val) + val = tonumber(val) + for i = 1, #bytes do + if bytes[i] == val then + break + end + end + while #bytes >= i do + table.remove(bytes) + end + return bytes +end + function env.byte_swap(bytes, val) local newbytes = {} val = tonumber(val) @@ -136,20 +506,19 @@ function env.bit_swap(bytes, swap) return bytes end -function env.bitmask(bytes, mask) - local newbytes = 0 - bytes = string.unpack(">I" .. #bytes, table.concat(bytes)) - for i = 1, #mask do - newbytes = newbytes | (((bytes >> mask.ishift) & mask.mask) << mask.oshift) - end - bytes = {} - while newbytes ~= 0 do - bytes[#bytes + 1] = newbytes & 0xff - newbytes = newbytes >> 8 - end - newbytes = {} - for i = 1, #bytes do - newbytes[i] = string.char(bytes[#bytes + 1 - i]) +function env.bitmask(bytes, masks) + local newbytes = {} + bytes = table.concat(bytes) + bytes = table.pack(string.unpack(string.rep("c1", #bytes), bytes)) + bytes[#bytes] = nil + for num, mask in ipairs(masks) do + newbytes[#newbytes + 1] = "" + for num2, bytemask in ipairs(mask.mask) do + local val = bytes[num2]:byte() & bytemask + if val ~= 0 then + newbytes[#newbytes] = newbytes[#newbytes] .. string.char(val) + end + end end return newbytes end @@ -166,24 +535,42 @@ function env.frombcd(val) end function env.basechar(bytes, base) - emu.print_verbose("data_hiscore: basechar " .. base .. " unimplemented\n") - if base == "32" then - elseif base == "40" then + if base == 32 or base == 40 then + local newbytes = {} + for num, char in ipairs(bytes) do + local nchar = (char:byte(1) << 8) + char:byte(2) + local pos = #newbytes + for i = 1, 3 do + table.insert(newbytes, pos + 1, string.char(nchar % base)) + nchar = nchar // base + end + end + return newbytes end + emu.print_verbose("data_hiscore: basechar " .. base .. " unimplemented") return bytes end -function env.charset_conv(bytes, charset) +function env.charset_conv(bytes, charset, aoffset) + if not aoffset then aoffset = 0 end if type(charset) == "string" then local chartype, offset, delta = charset:match("CS_(%w*)%[?(%-?%d?%d?),?(%d?%d?)%]?") + if offset then offset = tonumber(offset) else offset = 0 end + if delta then delta = tonumber(delta) else delta = 1 end if chartype == "NUMBER" then - + for num, char in ipairs(bytes) do + char = char:byte() - aoffset - offset + if char >= 48 and char <= 57 then + bytes[num] = string.char(char) + end + end + return bytes end - emu.print_verbose("data_hiscore: charset " .. chartype .. " unimplemented\n") + emu.print_verbose("data_hiscore: charset " .. chartype .. " unimplemented") return bytes end for num, char in ipairs(bytes) do - char = string.byte(char) + char = char:byte() - aoffset if charset[char] then bytes[num] = charset[char] elseif charset.default then @@ -207,6 +594,15 @@ function env.ascii_offset(bytes, offset) return bytes end +function env.index_from_value(col, index) + for i = 0, #col - 1 do + if col[i].val == index then + return i + end + end + return index +end + env.tostring = tostring env.type = type env.table = { pack = table.pack, concat = table.concat } @@ -228,10 +624,8 @@ function dat.check(set, softlist) if softlist then return nil end - local datpath local function xml_parse(file) local table - datpath = file:fullpath():gsub(".zip", "/") local data = file:read(file:size()) data = data:match("<hi2txt.->(.*)</ *hi2txt>") local function get_tags(str, parent) @@ -284,29 +678,23 @@ function dat.check(set, softlist) local function parse_table(xml) local total_size = 0 - local s = { "local data = open('" .. xml.structure[1].file .. "', size)\nlocal offset = 1\nlocal arr = {}", - "local elem, bytes, offset, value, lastindex, output"} + local s = { "local data = open('" .. xml.structure[1].file .. "', size, " .. (xml.structure[1]['byte-swap'] or "nil") .. ")\nlocal offset = 1\nlocal arr = {}", + "local elem, bytes, offset, value, index, output"} local fparam = {} if xml.bitmask then - local bitmask = "local bitmask = {" + local bitmask = "local bitmasks = {" for id, masks in pairs(xml.bitmask) do bitmask = bitmask .. "['" .. id .. "'] = {" for num, mask in ipairs(masks.character) do - mask = mask:gsub("( )", "") - local newmask = tonumber(mask, 2) - local shift = 0 - local count = 8 - while (newmask & 1) == 0 do - newmask = newmask >> 1 - shift = shift + 1 - end - if masks.byte-completion and masks.byte-completion == "no" then - count = 0 - while (newmask >> count) & 1 == 1 do - count = count + 1 + local bytemasks = {} + mask.mask:gsub("[01]*", function(s) + local bytemask = 0 + for i = 1, 8 do + bytemask = bytemask | (tonumber(s:sub(i, i)) << (8 - i)) end - end - bitmask = bitmask .. "{ mask = " .. newmask .. ", ishift = " .. shift .. ", oshift = " .. count .. "}," + bytemasks[#bytemasks + 1] = bytemask + end) + bitmask = bitmask .. "{ mask = {" .. table.concat(bytemasks, ", ") .. "}, complete = \"" .. (mask['byte-completion'] or "yes") .. "\"}" end bitmask = bitmask .. "}," end @@ -318,10 +706,19 @@ function dat.check(set, softlist) local default charset = charset .. "['" .. id .. "'] = {" for src, char in pairs(set.char) do + local subchar = char.dst:gsub("&([^;]*);", function(s) + if htmlentites[s] then + return htmlentites[s] + elseif s[1] == "#" then + return utf8.char(tonumber(s:sub(2))) + else + return s + end + end) if char.default and char.default == "yes" then - default = char.dst + default = subchar end - charset = charset .. "[" .. src .. "]" .. " = '" .. char.dst .. "'," + charset = charset .. "[" .. src .. "]" .. " = '" .. subchar .. "'," end if default then charset = charset .. "default = " .. default @@ -386,7 +783,7 @@ function dat.check(set, softlist) elseif first == "S" then local suffix = rest:match("uffix(.*)") if suffix then - oper = " val = val .. '" .. rest .. "'" + oper = " val = val .. '" .. suffix .. "'" end elseif (first == "h" and rest == "exadecimal_string") or (first == "0" and rest == "x") then oper = " val = string.format('0x%x', val)" @@ -396,7 +793,11 @@ function dat.check(set, softlist) if not oper then oper = " val = format['" .. form .. "'](val, {" for num1, colpar in ipairs(fparam[form]) do - oper = oper .. "arr['" .. colpar .. "'][i].val or arr['" .. colpar .. "'][1].val," + if fparam[form].full then + oper = oper .. "arr['" .. colpar .. "']," + else + oper = oper .. "arr['" .. colpar .. "'][i].val or arr['" .. colpar .. "'][0].val," + end end oper = oper .. "})" end @@ -406,15 +807,10 @@ function dat.check(set, softlist) end if xml.format then - local format = { "local format = {" } + local format = { "local format = {}" } for num, form in ipairs(xml.format) do local param = {} - format[#format + 1] = "['" .. form["id"] .. "'] = " - if form["input-as-subcolumns-input"] then - --format[#format + 1] = "input_as_subcolumns_input = '" .. form["input-as-subcolumns-input"] .. "'," - emu.print_verbose("data_hiscore: input-as-subcolumns-input unimplemented\n") - end - format[#format + 1] = "function(val, param) " + format[#format + 1] = "format['" .. form["id"] .. "'] = function(val, param)" if form["formatter"] then format[#format + 1] = "local function tempform(val) " end @@ -432,16 +828,34 @@ function dat.check(set, softlist) elseif op.tag == "multiply" then format[#format + 1] = "val = val * " .. op.text elseif op.tag == "divide" then - format[#format + 1] = "val = val / " .. op.text + if op.text then + format[#format + 1] = "val = val / " .. op.text + elseif op.field then + if op.field[1].format then + local fform = check_format(op.field[1].format) + format[#format + 1] = fform .. " val = val / tempform(val)" + end + else + format[#format + 1] = "val = 0" + for num2, col in ipairs(op) do + param[#param + 1] = col["id"] + if col["format"] then + local colform = check_format(col["format"]) + format[#format + 1] = colform .. " val = val / tempform(val)" + else + format[#format + 1] = "val = val / param[" .. #param .. "]" + end + end + end elseif op.tag == "sum" then format[#format + 1] = "val = 0" - for num2, col in ipairs(op) do + param.full = true + for num2, col in ipairs(op.column) do param[#param + 1] = col["id"] + format[#format + 1] = "for i = 0, #param[" .. #param .. "] do val = val + param[" .. #param .. "][i].val end" if col["format"] then local colform = check_format(col["format"]) format[#format + 1] = colform .. " val = val + tempform(val)" - else - format[#format + 1] = "val = val + param[" .. #param .. "]" end end elseif op.tag == "concat" then @@ -450,9 +864,13 @@ function dat.check(set, softlist) if col["tag"] == "txt" then format[#format + 1] = "val = val .. '" .. col["text"] .. "'" elseif col["format"] then - param[#param + 1] = col["id"] - local n = #param - format[#format + 1] = function() return " " .. check_format(col["format"]) .. " val = val .. tempform(param[" .. n .. "])" end + if col["id"] then + param[#param + 1] = col["id"] + local n = #param + format[#format + 1] = function() return " " .. check_format(col["format"]) .. " val = val .. tempform(param[" .. n .. "])" end + else + format[#format + 1] = function() return " " .. check_format(col["format"]) .. " val = val .. tempform(temp)" end + end else param[#param + 1] = col["id"] format[#format + 1] = "val = val .. param[" .. #param .. "]" @@ -514,10 +932,10 @@ function dat.check(set, softlist) elseif op.tag == "case" then format[#format + 1] = "val = temp" if not tonumber(op["src"]) then - op["src"] = "'" .. op["src"] .. "'" + op["src"] = "'" .. op["src"]:gsub("'", "\\'") .. "'" end if not tonumber(op["dst"]) then - op["dst"] = "'" .. op["dst"] .. "'" + op["dst"] = "'" .. op["dst"]:gsub("'", "\\'") .. "'" end if op["default"] == "yes" then format[#format + 1] = "local default = " .. op["dst"] @@ -531,7 +949,11 @@ function dat.check(set, softlist) op["operator"] = op["operator"]:gsub("<", "<") op["operator"] = op["operator"]:gsub(">", ">") end - format[#format + 1] = "if val " .. op["operator"] .. " " .. op["src"] .. " then" + if form["apply-to"] == "char" then + format[#format + 1] = "if val:byte() " .. op["operator"] .. " " .. op["src"] .. " then" + else + format[#format + 1] = "if val " .. op["operator"] .. " " .. op["src"] .. " then" + end format[#format + 1] = "val = " .. op["dst"] if op["format"] then format[#format + 1] = function() return " val = ".. check_format(col["operator-format"]) end @@ -549,7 +971,6 @@ function dat.check(set, softlist) if form["formatter"] then format[#format + 1] = "return string.format('" .. form["formatter"] .. "', tempform(val))\nend" end - format[#format + 1] = "," end for num, line in ipairs(format) do if type(line) == "string" then @@ -558,7 +979,6 @@ function dat.check(set, softlist) s[#s + 1] = line() end end - s[#s + 1] = "}" end local function parse_elem(elem, loopelem) local ret = 0 @@ -566,23 +986,15 @@ function dat.check(set, softlist) if elem["skip-first-bytes"] then s[#s + 1] = "offset = offset + " .. elem["skip-first-bytes"] end - s[#s + 1] = "for i = 1, " .. elem["count"] .. " do" + s[#s + 1] = "for i = 0, " .. elem["count"] - 1 .. " do" for num, elt in ipairs(elem) do - index = parse_elem(elt, elem) + parse_elem(elt, elem) end s[#s + 1] = "end" if elem["skip-last-bytes"] then s[#s + 1] = "offset = offset + " .. elem["skip-last-bytes"] end elseif elem["tag"] == "elt" then - s[#s + 1] = "if not arr['" .. elem["id"] .. "'] then arr['" .. elem["id"] .. "'] = {} end\nelem = {}" - s[#s + 1] = "bytes = table.pack(string.unpack('" .. string.rep("c1", elem["size"]) .. "', data, offset))" - if loopelem then - total_size = total_size + elem["size"] * loopelem["count"] - else - total_size = total_size + elem["size"] - end - s[#s + 1] = "offset = bytes[#bytes]\nbytes[#bytes] = nil" if elem["decoding-profile"] then if elem["decoding-profile"] == "base-40" then elem["src-unit-size"] = 16 @@ -604,6 +1016,23 @@ function dat.check(set, softlist) elem["base"] = "16" end end + local unitsize = 1 + if elem["src-unit-size"] then + local size = elem["src-unit-size"] / 8 + if size ~= math.floor(size) then + emu.print_verbose("src-unit-size " .. elem["src-unit-size"] .. " not suppoted)") + else + unitsize = math.floor(size) + end + end + s[#s + 1] = "if not arr['" .. elem["id"] .. "'] then arr['" .. elem["id"] .. "'] = {} end\nelem = {}" + s[#s + 1] = "bytes = table.pack(string.unpack('" .. string.rep("c" .. unitsize, elem["size"] / unitsize) .. "', data, offset))" + if loopelem then + total_size = total_size + elem["size"] * loopelem["count"] + else + total_size = total_size + elem["size"] + end + s[#s + 1] = "offset = bytes[#bytes]\nbytes[#bytes] = nil" local bytedec = elem["swap-skip-order"] or "byte-swap;bit-swap;byte-skip;endianness;byte-trim;nibble-skip;bitmask" local bytedecl = {} bytedec:gsub("([^;]*)", function(c) bytedecl[#bytedecl + 1] = c end) @@ -611,7 +1040,7 @@ function dat.check(set, softlist) if elem[func] then fixfunc = func:gsub("-", "_") if func == "bitmask" then - s[#s + 1] = "bytes = " .. fixfunc .. "(bytes, bitmask['" .. elem[func] .. "'])" + s[#s + 1] = "bytes = " .. fixfunc .. "(bytes, bitmasks['" .. elem[func] .. "'])" else s[#s + 1] = "bytes = " .. fixfunc .. "(bytes, '" .. elem[func] .. "')" end @@ -623,29 +1052,41 @@ function dat.check(set, softlist) if elem["base"] == "16" then s[#s + 1] = "value = frombcd(value)" end + if elem["format"] then + s[#s + 1] = check_format(elem["format"]) + s[#s + 1] = "value = tempform(value)" + end s[#s + 1] = "elem.val = value" elseif elem["type"] == "text" then + if elem["base"] then + s[#s + 1] = "bytes = basechar(bytes, " .. elem["base"] .. ")" + end if elem["ascii-step"] then s[#s + 1] = "bytes = ascii_step(bytes, " .. elem["ascii-step"] .. ")" end if elem["ascii-offset"] then s[#s + 1] = "bytes = ascii_offset(bytes, " .. elem["ascii-offset"] .. ")" end - if elem["base"] then - s[#s + 1] = "bytes = basechar(bytes, " .. elem["base"] .. ")" - end if elem["charset"] then local charsets = {} elem["charset"]:gsub("([^;]*)", function(s) charsets[#charsets + 1] = s return "" end) for num, charset in pairs(charsets) do if charset:match("^CS_") then - s[#s + 1] = "bytes = charset_conv(bytes, " .. charset .. ")" + s[#s + 1] = "bytes = charset_conv(bytes, \"" .. charset .. "\"" elseif charset ~= "" then - s[#s + 1] = "bytes = charset_conv(bytes, charset['" .. charset .. "'])" + s[#s + 1] = "bytes = charset_conv(bytes, charset['" .. charset .. "']" + end + if elem["ascii-offset"] then + s[#s] = s[#s] .. ", " .. elem["ascii-offset"] end + s[#s] = s[#s] .. ")" end end s[#s + 1] = "elem.val = table.concat(bytes)" + if elem["format"] then + s[#s + 1] = check_format(elem["format"]) + s[#s + 1] = "elem.val = tempform(elem.val)" + end end local index if elem["table-index"] then @@ -657,27 +1098,31 @@ function dat.check(set, softlist) local step = loopelem["step"] or 1 local ref, reftype if elem["table-index"] then - elem["table-index"]:match("([%w ]*):([%a_]*)") + ref, reftype = elem["table-index"]:match("([%w ]*):([%a_]*)") end if not elem["table-index"] or elem["table-index"] == "loop_index" then - index = "(i - 1) * " .. step .. " + " .. start + index = "i * " .. step .. " + " .. start elseif elem["table-index"] == "loop_reverse_index" then - index = "(" .. total .. "- i) * " .. step .. " + " .. start + index = "(" .. total .. "- i - 1) * " .. step .. " + " .. start elseif elem["table-index"] == "itself" then index = "value" - elseif elem["table-index"] == "last" then - index = "lastindex" elseif reftype then - index = reftype .. "(arr, '" .. ref .. "')" + index = reftype .. "(arr['" .. ref .. "'], i)" end end - if index then - s[#s + 1] = "elem.index = " .. index - s[#s + 1] = "lastindex = elem.index" + if elem["table-index-format"] and index then + s[#s + 1] = check_format(elem["table-index-format"]) + index = "tempform(" .. index .. ")" + end + if loopelem then + if index then + s[#s + 1] = "index = " .. index + end + s[#s + 1] = "arr['" .. elem["id"] .. "'][index] = elem" + else + s[#s + 1] = "arr['" .. elem["id"] .. "'] = elem" end - s[#s + 1] = "arr['" .. elem["id"] .. "'][#arr['" .. elem["id"] .. "'] + 1] = elem" end - return index end for num, elem in ipairs(xml.structure[1]) do if elem["tag"] == "loop" or elem["tag"] == "elt" then @@ -705,6 +1150,7 @@ function dat.check(set, softlist) local head = {} local dat = {} local loopcnt + local cols = 0 local igncol, ignval if fld["line-ignore"] then igncol, ignval = fld["line-ignore"]:match("([^:]*):(.*)") @@ -717,15 +1163,16 @@ function dat.check(set, softlist) if not col["src"] then col["src"] = col["id"] end - if not loopcnt and col["src"] ~= "index" then - table.insert(dat, 1, "for i = 1, #arr['" .. col["src"] .. "'] do") - table.insert(dat, 2, "local index = arr['" .. col["src"] .. "'][i].index or i - 1") - table.insert(dat, 3, "local line = ''") + if not loopcnt and col["src"] ~= "index" and col["src"] ~= "unsorted_index" then + table.insert(dat, 1, "for i = 0, #arr['" .. col["src"] .. "'] do") + table.insert(dat, 2, "local line = ''") loopcnt = true end head[#head + 1] = "output = output .. '" .. col["id"] .. "\\t'" if col["src"] == "index" then - dat[#dat + 1] = "value = index" + dat[#dat + 1] = "value = i" + elseif col["src"] == "unsorted_index" then + dat[#dat + 1] = "value = i" else dat[#dat + 1] = "if arr['" .. col["src"] .. "'] then value = arr['" .. col["src"] .. "'][i].val end" end @@ -737,15 +1184,18 @@ function dat.check(set, softlist) dat[#dat + 1] = "local checkval = value" end dat[#dat + 1] = "line = line .. value .. '\\t'" + cols = cols + 1 end end - if igncol then - dat[#dat + 1] = "if checkval ~= " .. ignval .. " then output = output .. line .. '\\n' end\nend" - else - dat[#dat + 1] = "output = output .. line .. '\\n'\nend" + if cols > 0 then + if igncol then + dat[#dat + 1] = "if checkval ~= " .. ignval .. " then output = output .. line .. '\\n' end\nend" + else + dat[#dat + 1] = "output = output .. line .. '\\n'\nend" + end + s[#s + 1] = table.concat(head, "\n") .. "\noutput = output .. '\\n'" + s[#s + 1] = table.concat(dat, "\n") end - s[#s + 1] = table.concat(head, "\n") .. "\noutput = output .. '\\n'" - s[#s + 1] = table.concat(dat, "\n") end end end @@ -753,7 +1203,7 @@ function dat.check(set, softlist) -- cache script local script = table.concat(s, "\n") - local scrpath = datpath .. "/" + local scrpath = path:match("[^;]*") .. "/" local scrfile = io.open(scrpath .. set .. ".lua", "w+") if not scrfile then lfs.mkdir(scrpath) @@ -767,7 +1217,7 @@ function dat.check(set, softlist) if curset == set then if output then - return _("High Scores") + return _p("plugin-data", "High Scores") else return nil end @@ -775,12 +1225,12 @@ function dat.check(set, softlist) output = nil curset = set - local scrfile = emu.file(lfs.env_replace(mame_manager:ui():options().entries.historypath:value():gsub("([^;]+)", "%1/hi2txt")), 1) + local scrfile = emu.file(path, 1) local ret = scrfile:open(set .. ".lua") local script if ret then function get_xml_table(fileset) - local file = emu.file(lfs.env_replace(mame_manager:ui():options().entries.historypath:value():gsub("([^;]+)", "%1/hi2txt")), 1) + local file = emu.file(path, 1) local ret = file:open(fileset .. ".xml") if ret then return nil @@ -815,7 +1265,7 @@ function dat.check(set, softlist) end end if output then - return _("High Scores") + return _p("plugin-data", "High Scores") else return nil end diff --git a/plugins/data/data_history.lua b/plugins/data/data_history.lua index 331243b57bf..b233405e984 100644 --- a/plugins/data/data_history.lua +++ b/plugins/data/data_history.lua @@ -1,18 +1,210 @@ local dat = {} + +local db = require('data/database') local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("history.dat", "## REVISION:") +local file = 'history.xml' +local tablename + +local function init() + -- check for old history table + if db.get_version('history.dat') then + db.exec([[DROP TABLE "history.dat";]]) + db.exec([[DROP TABLE "history.dat_idx";]]) + db.set_version('history.dat', nil) + end + + local fh, filepath, dbver + fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + ver = dbver + end + return + end + + -- scan file for version + for line in fh:lines() do + local match = line:match('<history([^>]*)>') + if match then + match = match:match('version="([^"]*)"') + if match then + ver = match + break + end + end + end + if (not ver) or (ver == dbver) then + fh:close() + ver = dbver + return + end + + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s_idx" ( + name VARCHAR NOT NULL, + list VARCHAR NOT NULL, + data INTEGER NOT NULL);]], + tablename)) + db.check(string.format('creating %s index table', file)) + db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename)) + db.check(string.format('creating %s data table', file)) + db.exec( + string.format( + [[CREATE INDEX "namelist_%s" ON "%s_idx" (name, list);]], + tablename, tablename)) + db.check(string.format('creating %s name/list index', file)) + end + + local slaxml = require('xml') + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check(string.format('starting %s transaction', file)) then + fh:close() + ver = dbver + return + end + + -- clean out previous data and update the version + if dbver then + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + db.exec(string.format([[DELETE FROM "%s_idx";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + end + db.set_version(file, ver) + if not db.check(string.format('updating %s version', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + + fh:seek('set') + local buffer = fh:read('a') + + local lasttag + local entry = {} + local rowid + + local dataquery = db.prepare( + string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename)) + local indexquery = db.prepare( + string.format([[INSERT INTO "%s_idx" (name, list, data) VALUES (?, ?, ?);]], tablename)) + + local parser = slaxml:parser{ + startElement = function(name) + lasttag = name + if name == 'entry' then + entry = {} + rowid = nil + elseif (name == 'system') or (name == 'item') then + table.insert(entry, {}) + end + end, + attribute = function(name, value) + if (name == 'name') or (name == 'list') then + entry[#entry][name] = value + end + end, + text = function(text, cdata) + if lasttag == 'text' then + text = text:gsub('\r', '') -- strip carriage returns + dataquery:bind_values(text) + while true do + local status = dataquery:step() + if status == db.DONE then + rowid = dataquery:last_insert_rowid(); + break + elseif result == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + -- FIXME: how to abort parse and roll back? + break + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + dataquery:reset() + end + end, + closeElement = function(name) + if (name == 'entry') and rowid then + for num, entry in pairs(entry) do + indexquery:bind_values(entry.name, entry.list or '', rowid) + while true do + local status = indexquery:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + -- FIXME: how to abort parse and roll back? + break + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + indexquery:reset() + end + end + end + } + + parser:parse(buffer, { stripWhitespace = true }) + dataquery:finalize() + indexquery:finalize() + fh:close() + + db.exec([[COMMIT TRANSACTION;]]) + if not db.check(string.format('committing %s transaction', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + ver = dbver + end +end + +if db then + init() +end function dat.check(set, softlist) - if not datread then + if not ver then return nil end - local status - status, info = pcall(datread, "bio", softlist or "info", set) - if not status or not info then - return nil + + info = nil + local query = db.prepare( + string.format( + [[SELECT f.data + FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid + WHERE fi.name = ? AND fi.list = ?;]], + tablename, tablename)) + query:bind_values(set, softlist or '') + while not info do + local status = query:step() + if status == db.ROW then + info = query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check(string.format('reading %s data', file)) + break + end end - return _("History") + query:finalize() + return info and _p('plugin-data', 'History') or nil end function dat.get() diff --git a/plugins/data/data_mameinfo.lua b/plugins/data/data_mameinfo.lua index 4ace7ba3f2a..0e77e328cef 100644 --- a/plugins/data/data_mameinfo.lua +++ b/plugins/data/data_mameinfo.lua @@ -1,23 +1,27 @@ local dat = {} -local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("mameinfo.dat", "# MAMEINFO.DAT", function(str) return str:gsub("\n\n", "\n") end) + +local info, ver +local datread = require('data/load_dat') +datread, ver = datread.open( + 'mameinfo.dat', + '# MAMEINFO.DAT', + function (str) return str:gsub('\n\n', '\n') end) function dat.check(set, softlist) if softlist or not datread then return nil end local status, drvinfo - status, info = pcall(datread, "mame", "info", set) + status, info = pcall(datread, 'mame', 'info', set) if not status or not info then return nil end - local sourcefile = emu.driver_find(set).source_file:match("[^/\\]*$") - status, drvinfo = pcall(datread, "drv", "info", sourcefile) + local sourcefile = emu.driver_find(set).source_file:match('[^/\\]*$') + status, drvinfo = pcall(datread, 'drv', 'info', sourcefile) if drvinfo then - info = info .. _("\n\n--- DRIVER INFO ---\nDriver: ") .. sourcefile .. "\n\n" .. drvinfo + info = info .. _p('plugin-data', '\n\n--- DRIVER INFO ---\nDriver: ') .. sourcefile .. '\n\n' .. drvinfo end - return _("MAMEinfo") + return _p('plugin-data', 'MAMEinfo') end function dat.get() diff --git a/plugins/data/data_marp.lua b/plugins/data/data_marp.lua index ed077a8c5bd..09fddf11ef9 100644 --- a/plugins/data/data_marp.lua +++ b/plugins/data/data_marp.lua @@ -1,124 +1,144 @@ -- get marp high score file from http://replay.marpirc.net/txt/scores3.htm local dat = {} + local db = require("data/database") local ver, info local function init() - local filepath - local dbver - local fh - local file = "scores3.htm" - - for path in mame_manager:ui():options().entries.historypath:value():gmatch("([^;]+)") do - filepath = lfs.env_replace(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end + local file = 'scores3.htm' - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading marp version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) - end - stmt:finalize() - - if not fh and dbver then - -- data in database but missing file, just use what we have - ver = dbver - return - elseif not fh then + local fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + ver = dbver + end return - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[" ( - romset VARCHAR NOT NULL, - data CLOB NOT NULL)]]) - db.check("creating marp table") - db.exec("CREATE INDEX \"romset_" .. file .. "\" ON \"" .. file .. "_idx\"(romset)") - db.check("creating marp index") end + -- scan file for version for line in fh:lines() do - local match = line:match("Top Scores from the MAME Action Replay Page %(([%w :]+)%)") + local match = line:match('Top Scores from the MAME Action Replay Page %(([%w :]+)%)') if match then ver = match break end end - - if not ver then + if (not ver) or (ver == dbver) then fh:close() + ver = dbver return end - if ver == dbver then + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s" ( + romset VARCHAR NOT NULL, + data CLOB NOT NULL, + PRIMARY KEY(romset));]], + tablename)) + db.check('creating MARP table') + end + + db.exec([[BEGIN TRANSACTION;]]) + if not db.check('starting MARP transaction') then fh:close() + ver = dbver return end + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting marp") - db.exec("DELETE FROM \"" .. file .. "_idx\"") - db.check("deleting marp index") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating marp version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting marp version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check('deleting previous MARP data') then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + end + end + if not db.set_version(file, ver) then + db.check('updating MARP version') + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning marp transation") + fh:seek('set') + local buffer = fh:read('a') local function gmatchpos() local pos = 1 - local set, data = "" + local set = '' + local data = '' local function iter() local lastset = set while true do local spos, scr, plyr, stype, ltype - local url = "" - spos, pos, set, stype, scr, plyr, ltype = buffer:find("\n%s*([%w_]*)%-?(%w-) :%s*(%d+) [;:] ([^:]+): [^%[\n]*%[?([%w ]*)[^\n]*", pos) + spos, pos, set, stype, scr, plyr, ltype = buffer:find('\n%s*([%w_]*)%-?(%w-) :%s*(%d+) [;:] ([^:]+): [^%[\n]*%[?([%w ]*)[^\n]*', pos) if not spos then return nil end - if set ~= "" then - if ltype ~= "" then - url = ltype .. "\t\n" - else - url = "" + + local url = '' + if set ~= '' then + if ltype ~= '' then + url = ltype .. '\t\n' + end + url = url .. 'http://replay.marpirc.net/r/' .. set + if stype ~= '' then + url = url .. '-' .. stype end - url = url .. "http://replay.marpirc.net/r/" .. set .. ((stype ~= "") and ("-" .. stype) or "") .. "\t\n" + url = url .. '\t\n' end - if set ~= "" and lastset and lastset ~= set then + + if (set ~= '') and (lastset ~= set) then local lastdata = data - data = url .. plyr .. "\t" .. scr .. "\n" + data = url .. plyr .. '\t' .. scr .. '\n' return lastset, lastdata end - data = data .. ((url ~= "") and ("\n" .. url) or "") .. plyr .. "\t" .. scr .. "\n" + + if url ~= '' then + data = data .. '\n' .. url + end + data = data .. plyr .. '\t' .. scr .. '\n' end end return iter end + local query = db.prepare( + string.format([[INSERT INTO "%s" (romset, data) VALUES (?, ?)]], tablename)) for set, data in gmatchpos() do - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?, ?)") - db.check("inserting marp values") - stmt:bind_values(set, data) - stmt:step() - stmt:finalize() + query:bind_values(set, data) + while true do + local status = query:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error('Database busy: inserting MARP data') + query:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + ver = dbver + return + elseif result ~= db.ROW then + db.check('inserting MARP data') + break + end + end + query:reset() end + query:finalize() + fh:close() - db.exec("END TRANSACTION") - db.check("ending marp transation") + db.exec([[COMMIT TRANSACTION;]]) + if not db.check('committing MARP transaction') then + db.exec([[ROLLBACK TRANSACTION;]]) + ver = dbver + end end if db then @@ -126,18 +146,26 @@ if db then end function dat.check(set, softlist) - if softlist or not ver or not db then + if softlist or (not ver) then return nil end + info = nil - local stmt = db.prepare("SELECT data FROM \"scores3.htm\" AS f WHERE romset = ?") - db.check("reading marp data") - stmt:bind_values(set) - if stmt:step() == db.ROW then - info = "#j2\n" .. stmt:get_value(0) + local query = db.prepare([[SELECT data FROM "scores3.htm" WHERE romset = ?;]]) + query:bind_values(set) + while not info do + local status = query:step() + if status == db.ROW then + info = '#j2\n' .. query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check('reading MARP data') + break + end end - stmt:finalize() - return info and _("MARPScore") or nil + query:finalize() + return info and _p('plugin-data', 'MARPScore') or nil end function dat.get() diff --git a/plugins/data/data_messinfo.lua b/plugins/data/data_messinfo.lua index b679763d940..3c94322474f 100644 --- a/plugins/data/data_messinfo.lua +++ b/plugins/data/data_messinfo.lua @@ -1,24 +1,27 @@ local dat = {} -local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("messinfo.dat", "# MESSINFO.DAT", function(str) return str:gsub("\n\n", "\n") end) +local ver, info +local datread = require('data/load_dat') +datread, ver = datread.open( + 'messinfo.dat', + '# MESSINFO.DAT', + function (str) return str:gsub('\n\n', '\n') end) function dat.check(set, softlist) if softlist or not datread then return nil end local status, drvinfo - status, info = pcall(datread, "mame", "info", set) + status, info = pcall(datread, 'mame', 'info', set) if not status or not info then return nil end - local sourcefile = emu.driver_find(set).source_file:match("[^/\\]*$") - status, drvinfo = pcall(datread, "drv", "info", sourcefile) + local sourcefile = emu.driver_find(set).source_file:match('[^/\\]*$') + status, drvinfo = pcall(datread, 'drv', 'info', sourcefile) if drvinfo then - info = info .. _("\n\n--- DRIVER INFO ---\nDriver: ") .. sourcefile .. "\n\n" .. drvinfo + info = info .. _p('plugin-data', '\n\n--- DRIVER INFO ---\nDriver: ') .. sourcefile .. '\n\n' .. drvinfo end - return _("MESSinfo") + return _p('plugin-data', 'MESSinfo') end function dat.get() diff --git a/plugins/data/data_story.lua b/plugins/data/data_story.lua index 92b579f9ecb..2e7b40c74f9 100644 --- a/plugins/data/data_story.lua +++ b/plugins/data/data_story.lua @@ -1,25 +1,27 @@ local dat = {} + local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("story.dat", "# version") +local datread = require('data/load_dat') +datread, ver = datread.open('story.dat', '# version') function dat.check(set, softlist) if softlist or not datread then return nil end - local status, data = pcall(datread, "story", "info", set) + local status, data = pcall(datread, 'story', 'info', set) if not status or not data then return nil end local lines = {} - data = data:gsub("MAMESCORE records : ([^\n]+)", "MAMESCORE records :\t\n%1", 1) - for line in data:gmatch("[^\n]*") do - if not (#lines == 0 and line == "") and not (lines[#lines] == "" and line == "") then - lines[#lines + 1] = line:gsub("^(.-)_+([0-9.]+)$", "%1\t%2") + data = data:gsub('MAMESCORE records : ([^\n]+)', 'MAMESCORE records :\t\n%1', 1) + for line in data:gmatch('[^\n]*') do + if (line ~= '') or ((#lines ~= 0) and (lines[#lines] ~= '')) then + line = line:gsub('^(.-)_+([0-9.]+)$', '%1\t%2') + table.insert(lines, line) end end - info = "#j2\n" .. table.concat(lines, "\n") - return _("Mamescore") + info = '#j2\n' .. table.concat(lines, '\n') + return _p('plugin-data', 'Mamescore') end function dat.get() diff --git a/plugins/data/data_sysinfo.lua b/plugins/data/data_sysinfo.lua index 8010ff854e0..1ba13605821 100644 --- a/plugins/data/data_sysinfo.lua +++ b/plugins/data/data_sysinfo.lua @@ -1,18 +1,19 @@ local dat = {} + local ver, info -local datread = require("data/load_dat") -datread, ver = datread.open("sysinfo.dat", "# This file was generated on") +local datread = require('data/load_dat') +datread, ver = datread.open('sysinfo.dat', '# This file was generated on') function dat.check(set, softlist) if softlist or not datread then return nil end local status - status, info = pcall(datread, "bio", "info", set) + status, info = pcall(datread, 'bio', 'info', set) if not status or not info then return nil end - return _("Sysinfo") + return _p('plugin-data', 'Sysinfo') end function dat.get() diff --git a/plugins/data/database.lua b/plugins/data/database.lua index 6eda6ab1507..fe2b9ce84bb 100644 --- a/plugins/data/database.lua +++ b/plugins/data/database.lua @@ -1,41 +1,159 @@ -local sql = require("lsqlite3") -local datfile = {} +local sql = require('lsqlite3') local db -local function check_db(msg) +local function check_db_result(msg) if db:errcode() > sql.OK then - emu.print_error("Error: " .. msg .. " (" .. db:errcode() .. " - " .. db:errmsg() .. ")\n") + emu.print_error(string.format('Error: %s (%s - %s)', msg, db:errcode(), db:errmsg())) + return false end + return true end -do - local dbpath = lfs.env_replace(mame_manager:ui():options().entries.historypath:value():match("([^;]+)")) - db = sql.open(dbpath .. "/history.db") - if not db then - lfs.mkdir(dbpath) - db = sql.open(dbpath .. "/history.db") +local function settings_path() + return manager.machine.options.entries.homepath:value():match('([^;]+)') .. '/data' +end + +local function check_version_table() + local found = false + db:exec( + [[SELECT COUNT(*) FROM sqlite_master WHERE name = 'version' AND type = 'table';]], + function (udata, cols, values, names) + found = tonumber(values[1]) > 0 + return 0 + end) + if check_db_result('checking for "version" table') and (not found) then + db:exec( + [[CREATE TABLE version ( + datfile VARCHAR NOT NULL, + version VARCHAR NOT NULL, + PRIMARY KEY (datfile));]]) + found = check_db_result('creating "version" table') + end + if not found then + db:close() + db = nil + end +end + +local function open_existing() + db = sql.open(settings_path() .. '/history.db', sql.OPEN_READWRITE) + if db then + check_version_table() + end + return db +end + +local function open_create() + -- first try to create settings directory + local dir = settings_path() + local attr = lfs.attributes(dir) + if (not attr) and (not lfs.mkdir(dir)) then + emu.print_error(string.format('Error creating data plugin settings directory "%s"', dir)) + elseif attr and (attr.mode ~= 'directory') then + emu.print_error(string.format('Error opening data plugin database: "%s" is not a directory', dir)) + else + -- now try to open the database + local dbpath = dir .. '/history.db' + db = sql.open(dbpath, sql.OPEN_READWRITE | sql.OPEN_CREATE) if not db then - emu.print_error("Unable to create history.db\n") - return false + emu.print_error(string.format('Error opening data plugin database "%s"', dbpath)) + else + check_version_table() end - check_db("opening database") end + return db end -if db then - local found = false - db:exec("select * from sqlite_master where name = 'version'", function() found = true return 0 end) - check_db("checking for 'version' table") - if not found then - db:exec([[ - CREATE TABLE version ( - version VARCHAR NOT NULL, - datfile VARCHAR UNIQUE NOT NULL)]]) - check_db("creating 'version' table") + +local dbtable = { + ROW = sql.ROW, + BUSY = sql.BUSY, + DONE = sql.DONE, + check = check_db_result } + +function dbtable.sanitize_name(name) + return name:gsub('[^%w%."]', '_') +end + +function dbtable.get_version(filename) + -- don't try to create the database here, just return nil if it doesn't exist + if (not db) and (not open_existing()) then + return nil + end + local query = db:prepare([[SELECT version FROM version WHERE datfile = ?;]]) + query:bind_values(filename) + local result + while result == nil do + local status = query:step() + if status == sql.ROW then + result = query:get_value(0) + elseif status ~= sql.BUSY then + break + end + end + query:finalize() + return result +end + +function dbtable.set_version(filename, version) + if (not db) and (not open_create()) then + return nil + end + local query + if version ~= nil then + query = db:prepare( + [[INSERT INTO version (datfile, version) VALUES (?1, ?2) + ON CONFLICT(datfile) DO UPDATE set version = ?2;]]) + query:bind_values(filename, version) + else + query = db:prepare([[DELETE FROM version WHERE datfile = ?1;]]) + query:bind_values(filename) + end + local result + while result == nil do + local status = query:step() + if status == sql.DONE then + result = true + elseif result ~= sql.ROW then + result = false + end + end + query:finalize() + return result +end + +function dbtable.prepare(...) + if (not db) and open_create() then + dbtable.prepare = function (...) return db:prepare(...) end + end + if db then + return db:prepare(...) + else + return nil end end -local dbtable = { prepare = function(...) return db:prepare(...) end, - exec = function(...) return db:exec(...) end, ROW = sql.ROW, check = check_db } +function dbtable.exec(...) + if (not db) and open_create() then + dbtable.exec = function (...) return db:exec(...) end + end + if db then + return db:exec(...) + else + return nil + end +end + +function dbtable.open_data_file(file) + local fh, filepath + for path in mame_manager.ui.options.entries.historypath:value():gmatch('([^;]+)') do + filepath = path .. '/' .. file + fh = io.open(filepath, 'r') + if fh then + break + end + end + return fh, filepath, dbtable.sanitize_name(file), dbtable.get_version(file) +end -return db and dbtable or false +return dbtable diff --git a/plugins/data/init.lua b/plugins/data/init.lua index 4ef3675968f..0ce3dfa4631 100644 --- a/plugins/data/init.lua +++ b/plugins/data/init.lua @@ -4,17 +4,21 @@ -- heading if it supports the set otherwise nil and get which returns the data -- the script should be named data_<name>.lua -- this is set default on in the plugin.json -local exports = {} -exports.name = "data" -exports.version = "0.0.1" -exports.description = "Data plugin" -exports.license = "The BSD 3-Clause License" -exports.author = { name = "Carl" } +local exports = { + name = 'data', + version = '0.0.2', + description = 'Data plugin', + license = 'BSD-3-Clause', + author = { name = 'Carl' } } local data = exports +local plugindir + +local reset_subscription + function data.set_folder(path) - data.path = path + plugindir = path end function data.startplugin() @@ -22,60 +26,66 @@ function data.startplugin() local valid_lst = {} local cur_set local cur_list - emu.register_start(function() - data_scr = {} - for file in lfs.dir(data.path) do - local name = string.match(file, "^(data_.*).lua$") - if name then - local script = require("data/" .. name) - if script then - data_scr[#data_scr + 1] = script + + reset_subscription = emu.add_machine_reset_notifier( + function () + data_scr = {} + for file in lfs.dir(plugindir) do + local name = string.match(file, '^(data_.*).lua$') + if name then + local script = require('data/' .. name) + if script then + table.insert(data_scr, script) + end + end end - end - end - end) - emu.register_callback(function(set) - local ret = {} - if set == "" then - set = emu.romname() - end - if set == cur_set then - return cur_list - end - cur_set = set - if not set then - return nil - end - valid_lst = {} - for num, scr in ipairs(data_scr) do - local setname, softname = set:match("^([^,]+),?(.*)$") - if softname == "" then - softname = nil - end - local name = scr.check(setname, softname) - if name then - ret[#ret + 1] = name - valid_lst[#valid_lst + 1] = scr - end - end - cur_list = ret - return ret - end, "data_list") + end) - emu.register_callback(function(num) - return valid_lst[num + 1].get() - end, "data") + emu.register_callback( + function (set) + local ret = {} + if set == '' then + set = emu.romname() + end + if set == cur_set then + return cur_list + end + cur_set = set + if not set then + return nil + end + valid_lst = {} + for num, scr in ipairs(data_scr) do + local setname, softname = set:match('^([^,]+),?(.*)$') + if softname == '' then + softname = nil + end + local name = scr.check(setname, softname) + if name then + table.insert(ret, name) + table.insert(valid_lst, scr) + end + end + cur_list = ret + return ret + end, + 'data_list') - emu.register_callback(function(num) - local ver - if valid_lst[num + 1].ver then - ver = valid_lst[num + 1].ver() - end - if ver then - return ver - end - return "" - end, "data_version") + emu.register_callback( + function (num) + return valid_lst[num + 1].get() + end, + 'data') + + emu.register_callback( + function (num) + local ver + if valid_lst[num + 1].ver then + ver = valid_lst[num + 1].ver() + end + return ver or '' + end, + 'data_version') end return exports diff --git a/plugins/data/load_dat.lua b/plugins/data/load_dat.lua index ce4ffed5c80..0438c5499ba 100644 --- a/plugins/data/load_dat.lua +++ b/plugins/data/load_dat.lua @@ -1,66 +1,55 @@ local datfile = {} -local db = require("data/database") -function datfile.open(file, vertag, fixupcb) - if not db then - return nil - end - local function read(tag1, tag2, set) +local db = require('data/database') + +local function readret(file, tablename) + local query = db.prepare( + string.format( + [[SELECT f.data + FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid + WHERE fi.type = ? AND fi.val = ? AND fi.romset = ?;]], + tablename, tablename)) + local function read(tag, val, set) + query:bind_values(tag, val, set) local data - local stmt = db.prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [[" - AS f WHERE fi.type = ? AND fi.val = ? AND fi.romset = ? AND f.rowid = fi.data]]) - db.check("reading " .. tag1 .. " - " .. tag2 .. " - " .. set) - stmt:bind_values(tag1, tag2, set) - if stmt:step() == db.ROW then - data = stmt:get_value(0) + while not data do + local status = query:step() + if status == db.ROW then + data = query:get_value(0) + elseif status == db.DONE then + break + elseif status ~= db.BUSY then + db.check(string.format('reading %s data', file)) + break + end end - stmt:finalize() + query:reset() return data end + return read +end - local ver, dbver - local filepath - local fh - for path in mame_manager:ui():options().entries.historypath:value():gmatch("([^;]+)") do - filepath = lfs.env_replace(path) .. "/" .. file - fh = io.open(filepath, "r") - if fh then - break - end - end - -- remove unsafe chars from file for use in sql statement - file = file:gsub("[^%w%._]", "") - - local stmt = db.prepare("SELECT version FROM version WHERE datfile = ?") - db.check("reading version") - stmt:bind_values(file) - if stmt:step() == db.ROW then - dbver = stmt:get_value(0) +function datfile.open(file, vertag, fixupcb) + if not db then + return nil end - stmt:finalize() - if not fh and dbver then - -- data in database but missing file, just use what we have - return read, dbver - elseif not fh then - return nil - elseif not dbver then - db.exec("CREATE TABLE \"" .. file .. [[_idx" ( - type VARCHAR NOT NULL, - val VARCHAR NOT NULL, - romset VARCHAR NOT NULL, - data INTEGER NOT NULL)]]) - db.check("creating index") - db.exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)") - db.check("creating table") - db.exec("CREATE INDEX \"typeval_" .. file .. "\" ON \"" .. file .. "_idx\"(type, val)") - db.check("creating typeval index") + local fh, filepath, tablename, dbver = db.open_data_file(file) + if not fh then + if dbver then + -- data in database but missing file, just use what we have + return readret(file, tablename), dbver + else + return nil + end end + local ver if vertag then + -- scan file for version for line in fh:lines() do - local match = line:match(vertag .. "%s*([^%s]+)") + local match = line:match(vertag .. '%s*(%S+)') if match then ver = match break @@ -68,102 +57,204 @@ function datfile.open(file, vertag, fixupcb) end end if not ver then - -- use file ctime for version - ver = tostring(lfs.attributes(filepath, "change")) + -- fall back to file modification time for version + ver = tostring(lfs.attributes(filepath, 'change')) end if ver == dbver then fh:close() - return read, dbver + return readret(file, tablename), dbver + end + + if not dbver then + db.exec( + string.format( + [[CREATE TABLE "%s_idx" ( + type VARCHAR NOT NULL, + val VARCHAR NOT NULL, + romset VARCHAR NOT NULL, + data INTEGER NOT NULL);]], + tablename)) + db.check(string.format('creating %s index table', file)) + db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename)) + db.check(string.format('creating %s data table', file)) + db.exec( + string.format( + [[CREATE INDEX "typeval_%s" ON "%s_idx" (type, val, romset);]], + tablename, tablename)) + db.check(string.format('creating %s type/value index', file)) end + db.exec([[BEGIN TRANSACTION;]]) + if not db.check(string.format('starting %s transaction', file)) then + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end + + -- clean out previous data and update the version if dbver then - db.exec("DELETE FROM \"" .. file .. "\"") - db.check("deleting") - db.exec("DELETE FROM \"" .. file .. "_idx\"") - db.check("deleting index") - stmt = db.prepare("UPDATE version SET version = ? WHERE datfile = ?") - db.check("updating version") - else - stmt = db.prepare("INSERT INTO version VALUES (?, ?)") - db.check("inserting version") + db.exec(string.format([[DELETE FROM "%s";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end + db.exec(string.format([[DELETE FROM "%s_idx";]], tablename)) + if not db.check(string.format('deleting previous %s data', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + return readret(file, tablename), dbver + end end - stmt:bind_values(ver, file) - stmt:step() - stmt:finalize() - - do - local inblock = false - fh:seek("set") - local buffer = fh:read("a") - db.exec("BEGIN TRANSACTION") - db.check("beginning transaction") - local function gmatchpos() - local pos = 1 - local function iter() - local tag1, tag2, data, start, inblock = false - while not data do - local spos, epos, match = buffer:find("\n($[^\n\r]*)", pos) - if not spos then - return nil - end - if match ~= "$end" and not inblock then - if not tag1 then - tag1 = match - else - tag2 = match - start = epos + 1 - inblock = true - end - elseif inblock == true then - data = buffer:sub(start, spos) - inblock = false + db.set_version(file, ver) + if not db.check(string.format('updating %s version', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end + + local dataquery = db.prepare( + string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename)) + local indexquery = db.prepare( + string.format( + [[INSERT INTO "%s_idx" (type, val, romset, data) VALUES (?, ?, ?, ?)]], + tablename)) + + fh:seek('set') + local buffer = fh:read('a') + + local function gmatchpos() + local pos = 1 + local function iter() + local tags, data + while not data do + local npos + local spos, epos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', pos) + if not spos then + return nil + end + npos, epos = buffer:find('[\n\r]$%w+%s*[\n\r]+', epos) + if not npos then + return nil + end + tags = buffer:sub(spos, epos) + spos, npos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', epos) + if not spos then + return nil + end + data = buffer:sub(epos, spos) + pos = spos + end + return tags, data + end + return iter + end + + for info, data in gmatchpos() do + local tags = {} + local infotype + info = info:gsub(utf8.char(0xfeff), '') -- remove byte order marks + data = data:gsub(utf8.char(0xfeff), '') + for s in info:gmatch('[\n\r]$([^\n\r]*)') do + if s:find('=', 1, true) then + local m1, m2 = s:match('([^=]*)=(.*)') + for tag in m1:gmatch('[^,]+') do + for set in m2:gmatch('[^,]+') do + table.insert(tags, { tag = tag, set = set }) end - pos = epos end - return tag1, tag2, data + else + infotype = s + break end - return iter end - for info1, info2, data in gmatchpos() do - local tag, set = info1:match("^%$([^%s=]+)=?([^%s]*)") - if set and set ~= "" then - local tags = {} - local sets = {} - tag:gsub("([^,]+)", function(s) tags[#tags + 1] = s end) - set:gsub("([^,]+)", function(s) sets[#sets + 1] = s end) - if #tags > 0 and #sets > 0 then - local tag1 = info2:match("^$([^%s]*)") - data = data:gsub("\r", "") -- strip crs - if fixupcb then - data = fixupcb(data) + + data = data:gsub('[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]', '\n') + data = data:gsub('[\n\r]$end%s*[\n\r].*', '') + + if (#tags > 0) and infotype then + data = data:gsub('\r', '') -- strip carriage returns + if fixupcb then + data = fixupcb(data) + end + + dataquery:bind_values(data) + local row + while true do + local status = dataquery:step() + if status == db.DONE then + row = dataquery:last_insert_rowid(); + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil end - stmt = db.prepare("INSERT INTO \"" .. file .. "\" VALUES (?)") - db.check("inserting values") - stmt:bind_values(data) - stmt:step() - local row = stmt:last_insert_rowid() - stmt:finalize() - for num1, tag2 in pairs(tags) do - for num2, set in pairs(sets) do - if fixupcb then - fixupcb(data) + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break + end + end + dataquery:reset() + + if row then + for num, tag in pairs(tags) do + indexquery:bind_values(infotype, tag.tag, tag.set, row) + while true do + local status = indexquery:step() + if status == db.DONE then + break + elseif status == db.BUSY then + emu.print_error(string.format('Database busy: inserting %s data', file)) + dataquery:finalize() + indexquery:finalize() + db.exec([[ROLLBACK TRANSACTION;]]) + fh:close() + if dbver then + return readret(file, tablename), dbver + else + return nil end - stmt = db.prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?, ?)") - db.check("inserting into index") - stmt:bind_values(tag1, tag2, set, row) - stmt:step() - stmt:finalize() + elseif result ~= db.ROW then + db.check(string.format('inserting %s data', file)) + break end end + indexquery:reset() end end end - db.exec("END TRANSACTION") - db.check("ending transaction") end + + dataquery:finalize() + indexquery:finalize() + fh:close() + db.exec([[COMMIT TRANSACTION;]]) + if not db.check(string.format('committing %s transaction', file)) then + db.exec([[ROLLBACK TRANSACTION;]]) + if dbver then + return readret(file, tablename), dbver + else + return nil + end + end - return read, ver + return readret(file, tablename), ver end return datfile |