summaryrefslogtreecommitdiffstatshomepage
path: root/plugins/data/load_dat.lua
blob: 0438c5499baf4b3822f07045b5ff69055513cfe8 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
local datfile = {}

local db = require('data/database')

local function readret(file, tablename)
	local query = db.prepare(
		string.format(
			[[SELECT f.data
				FROM "%s_idx" AS fi LEFT JOIN "%s" AS f ON fi.data = f.rowid
				WHERE fi.type = ? AND fi.val = ? AND fi.romset = ?;]],
			tablename, tablename))
	local function read(tag, val, set)
		query:bind_values(tag, val, set)
		local data
		while not data do
			local status = query:step()
			if status == db.ROW then
				data = query:get_value(0)
			elseif status == db.DONE then
				break
			elseif status ~= db.BUSY then
				db.check(string.format('reading %s data', file))
				break
			end
		end
		query:reset()
		return data
	end
	return read
end


function datfile.open(file, vertag, fixupcb)
	if not db then
		return nil
	end

	local fh, filepath, tablename, dbver = db.open_data_file(file)
	if not fh then
		if dbver then
			-- data in database but missing file, just use what we have
			return readret(file, tablename), dbver
		else
			return nil
		end
	end

	local ver
	if vertag then
		-- scan file for version
		for line in fh:lines() do
			local match = line:match(vertag .. '%s*(%S+)')
			if match then
				ver = match
				break
			end
		end
	end
	if not ver then
		-- fall back to file modification time for version
		ver = tostring(lfs.attributes(filepath, 'change'))
	end
	if ver == dbver then
		fh:close()
		return readret(file, tablename), dbver
	end

	if not dbver then
		db.exec(
			string.format(
				[[CREATE TABLE "%s_idx" (
					type VARCHAR NOT NULL,
					val VARCHAR NOT NULL,
					romset VARCHAR NOT NULL,
					data INTEGER NOT NULL);]],
				tablename))
		db.check(string.format('creating %s index table', file))
		db.exec(string.format([[CREATE TABLE "%s" (data CLOB NOT NULL);]], tablename))
		db.check(string.format('creating %s data table', file))
		db.exec(
			string.format(
				[[CREATE INDEX "typeval_%s" ON "%s_idx" (type, val, romset);]],
				tablename, tablename))
		db.check(string.format('creating %s type/value index', file))
	end

	db.exec([[BEGIN TRANSACTION;]])
	if not db.check(string.format('starting %s transaction', file)) then
		fh:close()
		if dbver then
			return readret(file, tablename), dbver
		else
			return nil
		end
	end

	-- clean out previous data and update the version
	if dbver then
		db.exec(string.format([[DELETE FROM "%s";]], tablename))
		if not db.check(string.format('deleting previous %s data', file)) then
			db.exec([[ROLLBACK TRANSACTION;]])
			fh:close()
			return readret(file, tablename), dbver
		end
		db.exec(string.format([[DELETE FROM "%s_idx";]], tablename))
		if not db.check(string.format('deleting previous %s data', file)) then
			db.exec([[ROLLBACK TRANSACTION;]])
			fh:close()
			return readret(file, tablename), dbver
		end
	end
	db.set_version(file, ver)
	if not db.check(string.format('updating %s version', file)) then
		db.exec([[ROLLBACK TRANSACTION;]])
		fh:close()
		if dbver then
			return readret(file, tablename), dbver
		else
			return nil
		end
	end

	local dataquery = db.prepare(
		string.format([[INSERT INTO "%s" (data) VALUES (?);]], tablename))
	local indexquery = db.prepare(
		string.format(
			[[INSERT INTO "%s_idx" (type, val, romset, data) VALUES (?, ?, ?, ?)]],
			tablename))

	fh:seek('set')
	local buffer = fh:read('a')

	local function gmatchpos()
		local pos = 1
		local function iter()
			local tags, data
			while not data do
				local npos
				local spos, epos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', pos)
				if not spos then
					return nil
				end
				npos, epos = buffer:find('[\n\r]$%w+%s*[\n\r]+', epos)
				if not npos then
					return nil
				end
				tags = buffer:sub(spos, epos)
				spos, npos = buffer:find('[\n\r]$[^=\n\r]*=[^\n\r]*', epos)
				if not spos then
					return nil
				end
				data = buffer:sub(epos, spos)
				pos = spos
			end
			return tags, data
		end
		return iter
	end

	for info, data in gmatchpos() do
		local tags = {}
		local infotype
		info = info:gsub(utf8.char(0xfeff), '') -- remove byte order marks
		data = data:gsub(utf8.char(0xfeff), '')
		for s in info:gmatch('[\n\r]$([^\n\r]*)') do
			if s:find('=', 1, true) then
				local m1, m2 = s:match('([^=]*)=(.*)')
				for tag in m1:gmatch('[^,]+') do
					for set in m2:gmatch('[^,]+') do
						table.insert(tags, { tag = tag, set = set })
					end
				end
			else
				infotype = s
				break
			end
		end

		data = data:gsub('[\n\r]$end%s*[\n\r]$%w+%s*[\n\r]', '\n')
		data = data:gsub('[\n\r]$end%s*[\n\r].-[\n\r]$%w+%s*[\n\r]', '\n')
		data = data:gsub('[\n\r]$end%s*[\n\r].*', '')

		if (#tags > 0) and infotype then
			data = data:gsub('\r', '') -- strip carriage returns
			if fixupcb then
				data = fixupcb(data)
			end

			dataquery:bind_values(data)
			local row
			while true do
				local status = dataquery:step()
				if status == db.DONE then
					row = dataquery:last_insert_rowid();
					break
				elseif status == db.BUSY then
					emu.print_error(string.format('Database busy: inserting %s data', file))
					dataquery:finalize()
					indexquery:finalize()
					db.exec([[ROLLBACK TRANSACTION;]])
					fh:close()
					if dbver then
						return readret(file, tablename), dbver
					else
						return nil
					end
				elseif result ~= db.ROW then
					db.check(string.format('inserting %s data', file))
					break
				end
			end
			dataquery:reset()

			if row then
				for num, tag in pairs(tags) do
					indexquery:bind_values(infotype, tag.tag, tag.set, row)
					while true do
						local status = indexquery:step()
						if status == db.DONE then
							break
						elseif status == db.BUSY then
							emu.print_error(string.format('Database busy: inserting %s data', file))
							dataquery:finalize()
							indexquery:finalize()
							db.exec([[ROLLBACK TRANSACTION;]])
							fh:close()
							if dbver then
								return readret(file, tablename), dbver
							else
								return nil
							end
						elseif result ~= db.ROW then
							db.check(string.format('inserting %s data', file))
							break
						end
					end
					indexquery:reset()
				end
			end
		end
	end

	dataquery:finalize()
	indexquery:finalize()

	fh:close()
	db.exec([[COMMIT TRANSACTION;]])
	if not db.check(string.format('committing %s transaction', file)) then
		db.exec([[ROLLBACK TRANSACTION;]])
		if dbver then
			return readret(file, tablename), dbver
		else
			return nil
		end
	end

	return readret(file, tablename), ver
end

return datfile