From d8854e4e5563265ba12c50cfcf9fb97cd37a2d40 Mon Sep 17 00:00:00 2001 From: Vas Crabb Date: Wed, 15 Jun 2022 05:07:09 +1000 Subject: Updated scripts to support SOURCES= builds after the reorganisation happens. --- scripts/build/makedep.py | 62 ++++++++- scripts/genie.lua | 6 +- scripts/src/main.lua | 65 +++++---- src/tools/reorganize.py | 352 ++++++++++++++++++++++++++--------------------- 4 files changed, 290 insertions(+), 195 deletions(-) diff --git a/scripts/build/makedep.py b/scripts/build/makedep.py index b4d47520c82..d9f92b08ac6 100755 --- a/scripts/build/makedep.py +++ b/scripts/build/makedep.py @@ -562,6 +562,7 @@ def parse_command_line(): subparser.add_argument('sources', metavar='', nargs='+', help='source files to include') subparser = subparsers.add_parser('sourcesfilter', help='generate driver filter for source files') + subparser.add_argument('-l', '--list', metavar='', required=True, help='master driver list file') subparser.add_argument('sources', metavar='', nargs='+', help='source files to include') subparser = subparsers.add_parser('driverlist', help='generate driver list source') @@ -662,9 +663,13 @@ def scan_source_dependencies(options): if ext.lower().startswith('.h'): components = components[:-1] test_siblings(components, base, depth) - if components == ('src', 'mame', 'includes'): - for aspect in ('audio', 'drivers', 'video', 'machine'): - test_siblings(('src', 'mame', aspect), base, depth) + if components[:2] == ('src', 'mame'): + if components[2:] == ('includes', ): + for aspect in ('audio', 'drivers', 'video', 'machine'): + test_siblings(('src', 'mame', aspect), base, depth) + else: + for aspect in ('_a', '_v', '_m'): + test_siblings(components, base + aspect, depth) handler = CppParser.Handler() handler.line = line_hook @@ -741,13 +746,60 @@ def write_project(options, projectfile, mappings, sources): def write_filter(options, filterfile): + def do_parse(p): + def line_hook(text): + text = text.strip() + if text.startswith('#'): + do_parse(os.path.join(os.path.dirname(n), text[1:].lstrip())) + elif text.startswith('@'): + parts = text[1:].lstrip().split(':', 1) + parts[0] = parts[0].strip() + if (parts[0] == 'source') and (len(parts) == 2): + parts[1] = parts[1].strip() + if not parts[1]: + sys.stderr.write('%s:%s: Empty source file name "%s"\n' % (p, parser.input_line, text)) + sys.exit(1) + else: + sources.add(parts[1]) + else: + sys.stderr.write('%s:%s: Unsupported directive "%s"\n' % (p, parser.input_line, text)) + sys.exit(1) + + n = os.path.normpath(p) + if n not in lists: + lists.add(n) + try: + listfile = io.open(n, 'r', encoding='utf-8') + except IOError: + sys.stderr.write('Unable to open list file "%s"\n' % (p, )) + sys.exit(1) + with listfile: + handler = CppParser.Handler() + handler.line = line_hook + parser = CppParser(handler) + try: + parser.parse(listfile) + except IOError: + sys.stderr.write('Error reading list file "%s"\n' % (p, )) + sys.exit(1) + except Exception as e: + sys.stderr.write('Error parsing list file "%s": %s\n' % (p, e)) + sys.exit(1) + + lists = set() + sources = set() + do_parse(options.list) + drivers = set() for source in options.sources: components = tuple(x for x in split_path(source) if x) - if (len(components) > 3) and (components[:3] == ('src', 'mame', 'drivers')): + if (len(components) > 3) and (components[:2] == ('src', 'mame')): ext = os.path.splitext(components[-1])[1].lower() if ext.startswith('.c'): - drivers.add('/'.join(components[3:])) + if components[2] == 'drivers': + drivers.add('/'.join(components[3:])) + elif '/'.join(components[2:]) in sources: + drivers.add('/'.join(components[2:])) for driver in sorted(drivers): filterfile.write(driver + '\n') diff --git a/scripts/genie.lua b/scripts/genie.lua index 76f18039000..dcffb53a0b6 100644 --- a/scripts/genie.lua +++ b/scripts/genie.lua @@ -557,7 +557,7 @@ if (_OPTIONS["PROJECT"] ~= nil) then end dofile (path.join(".." ,"projects", _OPTIONS["PROJECT"], "scripts", "target", _OPTIONS["target"],_OPTIONS["subtarget"] .. ".lua")) end -if (_OPTIONS["SOURCES"] == nil and _OPTIONS["PROJECT"] == nil) then +if (_OPTIONS["SOURCES"] == nil) and (_OPTIONS["PROJECT"] == nil) then if (not os.isfile(path.join("target", _OPTIONS["target"],_OPTIONS["subtarget"] .. ".lua"))) then error("File definition for TARGET=" .. _OPTIONS["target"] .. " SUBTARGET=" .. _OPTIONS["subtarget"] .. " does not exist") end @@ -1416,14 +1416,14 @@ if (_OPTIONS["SOURCES"] ~= nil) then local sourceargs = "" for word in string.gmatch(str, '([^,]+)') do if (not os.isfile(path.join(MAME_DIR, word))) then - print("File " .. word.. " does not exist") + print("File " .. word .. " does not exist") os.exit() end sourceargs = sourceargs .. " " .. word end OUT_STR = os.outputof( PYTHON .. " " .. MAME_DIR .. "scripts/build/makedep.py sourcesproject -r " .. MAME_DIR .. " -t " .. _OPTIONS["subtarget"] .. sourceargs ) load(OUT_STR)() - os.outputof( PYTHON .. " " .. MAME_DIR .. "scripts/build/makedep.py sourcesfilter" .. sourceargs .. " > ".. GEN_DIR .. _OPTIONS["target"] .. "/" .. _OPTIONS["subtarget"] .. ".flt" ) + os.outputof( PYTHON .. " " .. MAME_DIR .. "scripts/build/makedep.py sourcesfilter -l " .. MAME_DIR .. "src/" .. _OPTIONS["target"] .. "/" .. _OPTIONS["target"] .. ".lst" .. sourceargs .. " > ".. GEN_DIR .. _OPTIONS["target"] .. "/" .. _OPTIONS["subtarget"] .. ".flt" ) end group "libs" diff --git a/scripts/src/main.lua b/scripts/src/main.lua index da2ba71b61f..ec19bc578f6 100644 --- a/scripts/src/main.lua +++ b/scripts/src/main.lua @@ -285,45 +285,50 @@ if (STANDALONE~=true) then } }, } - else - if os.isfile(MAME_DIR .. "src/" .._target .. "/" .. _subtarget ..".lst") then - custombuildtask { + elseif os.isfile(MAME_DIR .. "src/" .._target .. "/" .. _subtarget ..".lst") then + custombuildtask { + { + MAME_DIR .. "src/" .. _target .. "/" .. _subtarget .. ".lst", + GEN_DIR .. _target .. "/" .. _subtarget .."/drivlist.cpp", + { MAME_DIR .. "scripts/build/makedep.py" }, { - MAME_DIR .. "src/" .. _target .. "/" .. _subtarget .. ".lst", - GEN_DIR .. _target .. "/" .. _subtarget .."/drivlist.cpp", - { MAME_DIR .. "scripts/build/makedep.py" }, - { - "@echo Building driver list...", - PYTHON .. " $(1) driverlist $(<) > $(@)" - } - }, - } - else - dependency { - { GEN_DIR .. _target .. "/" .. _target .."/drivlist.cpp", MAME_DIR .. "src/".._target .."/" .. _target ..".lst", true }, - } - custombuildtask { + "@echo Building driver list...", + PYTHON .. " $(1) driverlist $(<) > $(@)" + } + }, + } + else + dependency { + { GEN_DIR .. _target .. "/" .. _target .."/drivlist.cpp", MAME_DIR .. "src/".._target .."/" .. _target ..".lst", true }, + } + custombuildtask { + { + MAME_DIR .. "src/" .. _target .. "/" .. _target .. ".lst", + GEN_DIR .. _target .. "/" .. _target .. "/drivlist.cpp", + { MAME_DIR .. "scripts/build/makedep.py" }, { - MAME_DIR .. "src/" .. _target .. "/" .. _target .. ".lst", - GEN_DIR .. _target .. "/" .. _target .. "/drivlist.cpp", - { MAME_DIR .. "scripts/build/makedep.py" }, - { - "@echo Building driver list...", - PYTHON .. " $(1) driverlist $(<) > $(@)" - } - }, - } - end + "@echo Building driver list...", + PYTHON .. " $(1) driverlist $(<) > $(@)" + } + }, + } end end if (_OPTIONS["SOURCES"] ~= nil) then dependency { - { - GEN_DIR .. _target .. "/" .. _subtarget .."/drivlist.cpp", MAME_DIR .. "src/".._target .."/" .. _target ..".lst", true }, + { GEN_DIR .. _target .. "/" .. _subtarget .."/drivlist.cpp", MAME_DIR .. "src/".._target .."/" .. _target ..".lst", true }, } custombuildtask { - { GEN_DIR .. _target .."/" .. _subtarget ..".flt" , GEN_DIR .. _target .. "/" .. _subtarget .."/drivlist.cpp", { MAME_DIR .. "scripts/build/makedep.py", MAME_DIR .. "src/".._target .."/" .. _target ..".lst" }, {"@echo Building driver list...", PYTHON .. " $(1) driverlist $(2) -f $(<) > $(@)" }}, + { + GEN_DIR .. _target .."/" .. _subtarget ..".flt" , + GEN_DIR .. _target .. "/" .. _subtarget .."/drivlist.cpp", + { MAME_DIR .. "scripts/build/makedep.py", MAME_DIR .. "src/".._target .."/" .. _target ..".lst" }, + { + "@echo Building driver list...", + PYTHON .. " $(1) driverlist $(2) -f $(<) > $(@)" + } + }, } end diff --git a/src/tools/reorganize.py b/src/tools/reorganize.py index a5cac6646b8..c013dc4368a 100755 --- a/src/tools/reorganize.py +++ b/src/tools/reorganize.py @@ -1,163 +1,201 @@ #!/usr/bin/python -import sys +import argparse +import glob import os +import os.path +import re +import subprocess +import sys -if len(sys.argv) != 2: - print("Usage\n%s mame-cmake" % sys.argv[0]) - sys.exit(0) - -# Build the lists of files - -src = sys.argv[1] + '/src/mame/' - -for root, subdirs, files in os.walk(src + 'drivers'): - assert(len(subdirs) == 0) - srcd = files - -for root, subdirs, files in os.walk(src + 'audio'): - assert(len(subdirs) == 0) - srca = files - -for root, subdirs, files in os.walk(src + 'video'): - assert(len(subdirs) == 0) - srcv = files - -for root, subdirs, files in os.walk(src + 'machine'): - assert(len(subdirs) == 0) - srcm = files - -for root, subdirs, files in os.walk(src + 'includes'): - assert(len(subdirs) == 0) - srci = files - - -# Compute the future name of the files to avoid collisions -present = {} -renaming = {} - -def add(l, c): - global present, renaming - for f in l: - if f in present: - nfx = f.split('.') - assert(len(nfx) == 2) - nf = nfx[0] + '_' + c + '.' + nfx[1] -# print("%s -> %s" % (f, nf)) - assert(nf not in present) - else: - nf = f - present[nf] = True - renaming[c + '/' + f] = nf - -add(srci, "i") -add(srcd, "d") -add(srca, "a") -add(srcv, "v") -add(srcm, "m") - - -# Update the .lua files, lookup the per-file target names - -kl = { - 'includes': 'i', - 'drivers': 'd', - 'audio': 'a', - 'video': 'v', - 'machine' : 'm' -} - -keys = ['createMAMEProjects', 'createMESSProjects'] -fpmap = {'v/konamiic.txt': 'konami' } - -bad = False -for scr in ['arcade.lua', 'mess.lua', 'ci.lua', 'dummy.lua', 'nl.lua', 'tiny.lua', 'virtual.lua']: - fname = sys.argv[1] + '/scripts/target/mame/' + scr - lines = open(fname, 'r').readlines() - cp = None - f = open(fname, 'w') - for l in lines: - ls = l.rstrip('\r\n') - for kk in keys: - p = ls.find(kk) - if p == 0 or p == 1: - p = ls.find('"')+1 - p2 = ls.find('"', p) - cp = ls[p:p2] - if cp == 'mameshared': - cp = None - p = ls.find('MAME_DIR .. "src/mame/') - if p != -1: - p2 = ls.find('"', p+22) - fn = ls[p+22:p2] - fx = fn.split('/') - k = kl[fx[0]] + '/' + fx[1] - if cp != None: - if k in fpmap: - print("Duplicated: %s (%s and %s)" % (fn, cp, fpmap[k])) - bad = True - fpmap[k] = cp - ap = cp + +def parse_command(): + parser = argparse.ArgumentParser() + parser.add_argument('srcroot', metavar='', help='MAME source root directory') + return parser.parse_args() + + +def resolve_conflicts(root, *parts): + def walk(subdir): + for path, subdirs, files in os.walk(os.path.join(root, subdir)): + if len(subdirs): + raise Exception('Found %d unwanted subdirectories in %s' % (len(subdirs), path)) + return files + + present = set() + result = {} + for part in parts: + c = part[0] + for file in walk(part): + if file in present: + base, ext = os.path.splitext(file) + if not ext: + raise Exception('File %s has no extension' % (file, )) + nf = base + '_' + c + ext + #print('%s -> %s' % (os.path.join(part, file), nf)) + if nf in present: + raise Exception('File %s still conflicted after renaming from %s' % (nf, file)) else: - if k not in fpmap: - bad = True -# print("Missing in projects and seen in another .lua: %s" % fn) - ap = '?' - else: - ap = fpmap[k] - ls = ls[:p+22] + ap + '/' + renaming[k] + ls[p2:] - print(ls, file=f) - -for k in renaming.keys(): - if k not in fpmap: - print("Missing in projects: %s" % k) - bad = True - -if bad: - sys.exit(1) - -# Create the new subdirectories -for d in set(fpmap.values()): - os.mkdir(src + d) - -# Move the files around, patching the include paths as needed - -ikl = {} -for p, a in kl.items(): - ikl[a] = p - -for bfs, bfd in renaming.items(): - bfsx = bfs.split('/') - fs = src + ikl[bfsx[0]] + '/' + bfsx[1] - fdm = fpmap[bfs] - fd = src + fdm + '/' + bfd - fdh = open(fd, 'w') - for l in open(fs): - p0 = 0 - while l[p0] == ' ' or l[p0] == '\t': - p0 += 1 - if l[p0:p0+8] == '#include': - itype = '"' - si = l.find('"') - if si != -1: - si += 1 - ei = l.find('"', si) - fi = l[si:ei] - fix = fi.split('/') - fik = None - if len(fix) == 2 and fix[0] in kl: - fik = kl[fix[0]] + '/' + fix[1] - elif len(fix) == 1: - fik = bfsx[0] + '/' + fi - if fik != None and fik in fpmap: - fim = fpmap[fik] - if fim == fdm: - nfi = renaming[fik] + nf = file + present.add(nf) + result[part + '/' + file] = nf + + return result + + +def update_projects(root, renaming, *scripts): + keys = ('createMAMEProjects', 'createMESSProjects') + result = { 'video/konamiic.txt': 'konami' } + + bad = False + + for script in scripts: + fname = os.path.join(root, 'scripts', 'target', 'mame', script) + with open(fname, 'r', encoding='utf-8') as infile: + lines = infile.readlines() + cp = None + with open(fname, 'w', encoding='utf-8') as f: + for l in lines: + ls = l.rstrip('\r\n') + for kk in keys: + p = ls.find(kk) + if p == 0 or p == 1: + p = ls.find('"')+1 + p2 = ls.find('"', p) + cp = ls[p:p2] + if cp == 'mameshared': + cp = None + p = ls.find('MAME_DIR .. "src/mame/') + if p != -1: + p2 = ls.find('"', p + 22) + fn = ls[p+22:p2] + if cp is not None: + if fn in result: + print("Duplicated: %s (%s and %s)" % (fn, cp, result[fn])) + bad = True + result[fn] = cp + ap = cp else: - nfi = fim + '/' + renaming[fik] - l = l[:si] + nfi + l[ei:] - fdh.write(l) - os.remove(fs) - -for d in list(kl.keys()): - os.rmdir(src + d) + if fn not in result: + bad = True + #print("Missing in projects and seen in another .lua: %s" % fn) + ap = '?' + else: + ap = result[fn] + ls = ls[:p+22] + ap + '/' + renaming[fn] + ls[p2:] + print(ls, file=f) + + for k in renaming: + if k not in result: + print('Missing in projects: %s' % (k, )) + bad = True + + if bad: + raise Exception('Error updating project scripts') + + return result + + +def update_driver_list(fname, renaming, projectmap): + with open(fname, 'r', encoding='utf-8') as infile: + lines = infile.readlines() + with open(fname, 'w', encoding='utf-8') as outfile: + for l in lines: + ls = l.rstrip('\r\n') + match = re.match('(.*@source:)([-_0-9a-z]+\\.cpp)\\b(\s*)(.*)', ls) + if match: + f = 'drivers/' + match.group(2) + r = projectmap[f] + '/' + renaming.get(f, match.group(2)) + if match.group(3): + w = len(match.group(2)) + len(match.group(3)) + if len(r) < w: + r = r + (' ' * (w - len(r))) + else: + r = r + ' ' + print(match.group(1) + r + match.group(4), file=outfile) + else: + print(ls, file=outfile) + + +def update_driver_filter(fname, renaming, projectmap): + with open(fname, 'r', encoding='utf-8') as infile: + lines = infile.readlines() + with open(fname, 'w', encoding='utf-8') as outfile: + for l in lines: + ls = l.rstrip('\r\n') + match = re.match('(.*)(?