summaryrefslogtreecommitdiffstatshomepage
path: root/scripts/build/msgmerge.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/build/msgmerge.py')
-rw-r--r--scripts/build/msgmerge.py675
1 files changed, 675 insertions, 0 deletions
diff --git a/scripts/build/msgmerge.py b/scripts/build/msgmerge.py
new file mode 100644
index 00000000000..82fe3efdb97
--- /dev/null
+++ b/scripts/build/msgmerge.py
@@ -0,0 +1,675 @@
+#! /usr/bin/env python
+# -*- coding: iso-8859-1 -*-
+#
+# Copyright Terje Røsten <terjeros@phys.ntnu.no> Nov. 2003.
+#
+'''Merge two Uniforum style .po files together.
+
+This is a implementation (not complete) in Python of the GNU
+msgmerge(1) program. It can be used on the command line (or as a Python
+module).
+
+Usage: msgmerge.py [OPTIONS] def.po ref.pot
+
+The def.po file is an existing PO file with translations. The ref.pot
+file is the last created PO file with up-to-date source references but
+old translations, or a PO Template file.
+
+Options:
+ -U, --update update def.po,
+ do nothing if def.po is already up to date.
+ -o, --output-file=FILE write output to file FILE. Output is written to
+ stdout if set to - or if the option is not present.
+ -D, --docstrings don\'t remove docstring flag.
+ -h, --help display help text and exit.
+ -V, --version display version and exit.
+ -q, --quiet, --silent suppress progress indicators.
+'''
+from __future__ import generators
+
+if not __name__ == '__main__':
+ __doc__ += '''\
+
+When used as module the interesting functions are merge() and
+merge_dir().
+
+The merge() function does the same as the command line version, and
+the arguments are as follows. The first argument is the def.po file,
+then the ref.pot file. The third argument controls whether do work in
+update mode or not, then the next argument sets the output file. Set
+the next argument to False to remove docstring flags. The last
+argument can be used to suppress progress indicators. The default is
+to work in update mode with progress indicators.
+
+Example:
+ merge("def.po", "ref.pot")
+ merge the files def.po and ref.pot and write output to def.po if
+ there are any changes.
+ merge("def.po", "red.pot", docstrings = False, verbose = False,
+ update = False, outfile = "-")
+ merge the files def.po and ref.pot and write output to stdout,
+ remove docstring flag and be quiet.
+
+The merge_dir() function is useful when merging a directory of po
+files. The only required argument is the name of the directory with po
+files and the pot file. It will use simple glob to find the files. The
+second argument can be used to specify the pot file (in the
+directory). Third argument is a list of po files (then globbing will
+not be used) and the next argument is list of filename to exclude. The
+last argument can be used to suppress progress indicators. Docstring
+flag will not be removed.
+
+Example:
+ merge_dir("po")
+ merge (and update) all po files in directory po with the single pot
+ file in the same directory.
+
+The module raises the MsgmergeError exception in case of error.
+'''
+__revision__ = '$Id: msgmerge.py,v 1.41 2003/11/18 19:10:42 terjeros Exp $'
+__version__ = '0.1'
+name = 'msgmerge.py'
+
+__all__ = [ 'merge', 'merge_dir', 'MsgmergeError' ]
+
+import sys
+import re
+import string
+import getopt
+import difflib
+import glob
+import os.path
+import codecs
+
+try:
+ True, False
+except NameError:
+ True, False = 1, 0
+
+class Msgs:
+ '''Class to hold information about messages.'''
+ width = 80
+ file = ''
+ def __init__(self, msgid, msgstr, flag, lno, entry, **kwds):
+ self.id = msgid
+ self.str = msgstr
+ self.cmt = kwds.get('cmt', '')
+ self.ref = kwds.get('ref', '')
+ self.autocmt = kwds.get('autocmt', '')
+ self.flag = flag
+ self.entry = entry
+ self.lno = lno
+ self.count = 0
+ def wash(self):
+ self.id = wash(self.id, width = self.width,
+ filename = self.file, lno = self.lno)
+ self.str = wash(self.str, 'msgstr', width = self.width,
+ filename = self.file, lno = self.lno)
+ def used(self):
+ self.count += 1
+ def get_clean_id(self):
+ return self.id.replace('msgid "','', 1)
+ def obsolete(self):
+ self.width -= len('#~ ')
+ self.wash()
+ t = [ '#~ %s\n' % s for s in self.id.splitlines() ]
+ self.id = ''.join(t)
+ t = [ '#~ %s\n' % s for s in self.str.splitlines() ]
+ self.str = ''.join(t)
+
+class Options:
+ '''Class to hold options'''
+ def __init__(self, cmdline = False, **kwds):
+ if not cmdline:
+ self.update = kwds.get('update', True)
+ self.outfile = kwds.get('outfile', '-')
+ self.docstrings = kwds.get('docstrings', True)
+ self.verbose = kwds.get('verbose', False)
+ self.suffix = kwds.get('suffix', '~')
+ self.backup = kwds.get('backup', True)
+ else:
+ self.update = False
+ self.outfile = False
+ self.docstrings = False
+ self.verbose = True
+ self.suffix = '~'
+ self.backup = True
+
+class MsgmergeError(Exception):
+ '''Exception class for msgmerge'''
+
+def gen(lines):
+ '''
+ Generator which returns a line (with the obsolete prefix removed)
+ from the list of lines in <lines>, the line number is also
+ returned.
+ '''
+ lno = 0
+ for l in lines:
+ lno += 1
+ yield l.replace('#~ ', '', 1), lno
+ yield l, lno
+
+def slurp(s, g, sign):
+ '''
+ The string returned from iterator <g>\'s next() method is added to
+ the string <s> if string returned is beginning with the string
+ <sign>. The return value is the first returned string which do not
+ start with <sign>, the line number, the iterator <g> and the
+ (possibly) updated string <s>.
+ '''
+ l, lno = g.next()
+ while l.startswith(sign) or (sign == '# ' and l.strip() == '#'):
+ s += l
+ l, lno = g.next()
+ return l, lno, g, s
+
+def splitted_fit(chunk, line, width, break_always, break_after_space):
+ '''
+ Check if string <chunk> can be splitted by newline to fit into
+ string <line> with width smaller than <width>. The return value is
+ a tuple where the first element is the part of chunk which fits
+ and the second element is the rest of chunk.
+ '''
+ ret = '', chunk
+ l = len(chunk)
+ for i in range(l - 1, -1, -1):
+ if chunk[i] in break_always and len(chunk[0:i] + line) <= width:
+ ret = chunk[0:i], chunk[i:]
+ break
+ elif chunk[i] in break_after_space and i and chunk[i-1].strip() == '':
+ ret = chunk[0:i], chunk[i:]
+ break
+ elif chunk[i] == '\\' and len(chunk[i:]) > 1 and chunk[i+1] == '"' \
+ and len(chunk[0:i] + line) <= width:
+ ret = chunk[0:i], chunk[i:]
+ break
+ return ret
+
+def wrap(msg, width):
+ '''
+ Accept a list <msg> of strings to wrap, each string is wrapped to
+ width <width> and surrounded with a pair of ". The return value is
+ a string with these wrapped strings joined together with newlines.
+ '''
+ if msg.isspace() or not msg:
+ return '"%s"' % msg
+
+ # \ and " is here, but " is special in po files.
+ break_always = '$%+({['
+ # XXX what about: « © » ¦ § etc?
+ break_after_space = '_-=^`~\'<|>&*#@'
+ enders = '.:,;!?/])}|%-'
+ extra = string.punctuation
+ for c in enders:
+ extra = extra.replace(c, '')
+ escaped = { 'enders' : re.escape(enders),
+ 'extra' : re.escape(extra) }
+ regex = r'([\w%(extra)s]*[\s%(enders)s)]+[\s%(enders)s]*)' % escaped
+ r = re.compile(regex, re.UNICODE)
+ msg = [ m for m in r.split(msg) if not m == '']
+
+ lines = []
+ line = msg.pop(0)
+
+ # Handle \n on end of line
+ if len(msg) > 1 and msg[-1] == 'n' and len(msg[-2]) > 0 \
+ and msg[-2][-1] == '\\':
+ msg[-2] += msg[-1]
+ msg.pop()
+ # Do not allow a single \n on a line
+ if len(msg) > 2 and msg[-1] == '\\n':
+ msg[-2] += msg[-1]
+ msg.pop()
+
+ for m in msg:
+ if len(line) > width or len(m) > width or len(line + m) > width:
+ fit, rest = splitted_fit(m, line, width, break_always,
+ break_after_space)
+ line += fit
+ lines.append(line)
+ line = rest
+ else:
+ line += m
+ lines.append(line)
+ lines = [ '"%s"' % l for l in lines ]
+ return '\n'.join(lines)
+
+def normalize(lines):
+ '''
+ Normalize <lines>: e.g "\n\nText\n\n" becomes:
+ "\n"
+ "\n"
+ "Text\n"
+ "\n"
+ '''
+ if 0 < lines.find('\\n') < len(lines) - 3:
+ if lines[-3:] == '\\n"':
+ lines = lines[:-3].replace('\\n','\\n"\n"').replace('""\n','') \
+ + '\\n"'
+ else:
+ lines = lines.replace('\\n','\\n"\n"').replace('""\n','')
+ return lines
+
+def wash(msg, idx = 'msgid', width = 80, **kwds):
+ '''
+ Do washing on the msgstr or msgid fields. Wrap the text to fit in
+ width <width>. <msg> is a list of lines that makes up the field.
+ <idx> indicate msgid or msgstr, <width> holds the width. <filename>
+ and <lno> (line number) is picked up from <kwds>.
+ Returns the washed field as a string.
+ '''
+ msg = normalize(msg)
+ lines = msg.splitlines()
+ size = len(lines)
+ if size > 1 or len(msg) > width:
+ washed = []
+ # The first line is special
+ m = re.match('^%s "(.*)"$' % (idx, ), lines[0])
+ if not m:
+ print lines[0]
+ kwds['lno'] -= size + 1
+ raise MsgmergeError('parse error: %(filename)s:%(lno)s.'
+ % kwds)
+ washed.append(m.group(1))
+ if m.group(1).endswith(r'\n'):
+ washed.append('')
+ i = 0
+ for line in lines[1:]:
+ m = re.match('^"(\s*.*)"$', line)
+ i += 1
+ if not m:
+ print line
+ kwds['lno'] -= size - i + 1
+ raise MsgmergeError('parse error: %(filename)s:%(lno)s.'
+ % kwds)
+ washed[-1] += m.group(1)
+ if m.group(1).endswith(r'\n'):
+ washed.append('')
+ if washed[0] == '':
+ washed.pop(0)
+ if washed[-1] == '':
+ washed.pop()
+
+ washed = [ wrap(w, width - 3) for w in washed ] # " and \n removed.
+
+ # One line or multiline
+ if len(washed) == 1 and len('%s %s\n' % (idx, washed[0])) < width:
+ washed = '%s %s\n' % (idx, washed[0])
+ else:
+ washed = '%s ""\n%s\n' % (idx, '\n'.join(washed))
+ else:
+ washed = msg
+
+ return washed
+
+def parse(filename, entry):
+ '''
+ Parse po or pot file with name <filename>. Set the variable
+ <entry> to msgid/msgstr to indicate pot/po file. The return value
+ is a dict with msgid (washed) as key and Msgs instances as
+ values.
+ '''
+ lines = io(filename).readlines()
+ Msgs.file = filename
+ messages = {}
+ last = len(lines)
+ g = gen(lines)
+ cmt = autocmt = ref = flag = ''
+ msgid = False
+ lno = 0
+ while not lno == last:
+ l, lno = g.next()
+ if l.startswith('# '):
+ l, lno, g, cmt = slurp(l, g, '# ')
+ if l.startswith('#.'):
+ l, lno, g, autocmt = slurp(l, g, '#.')
+ if l.startswith('#:'):
+ l, lno, g, ref = slurp(l, g, '#:')
+ if l.startswith('#,'):
+ l, lno, g, flag = slurp(l, g, '#,')
+ if l.startswith('msgid'):
+ l, lno, g, msgid = slurp(l, g, '"')
+ if l.startswith('msgstr'):
+ l, lno, g, msgstr = slurp(l, g, '"')
+
+ if not lno == last and not l.strip() == '':
+ raise MsgmergeError('parse error: %s:%s.' % (filename, lno))
+
+ if msgid and entry == 'msgstr':
+ idx = wash(msgid, filename = filename, lno = lno)
+ messages[idx] = Msgs(msgid, msgstr, flag, lno, entry, cmt = cmt)
+ msgid = False; msgstr = cmt = autocmt = ref = flag = ''
+ elif msgid and entry == 'msgid':
+ idx = wash(msgid, filename = filename, lno = lno)
+ messages[idx] = Msgs(msgid, msgstr, flag, lno, entry,
+ autocmt = autocmt, ref = ref)
+ msgid = False; msgstr = cmt = autocmt = ref = flag = ''
+
+ for m in messages.values():
+ m.wash()
+ return messages
+
+def fuzzy_match(pot, defs):
+ '''
+ Try to find the best difflib match (with ratio > 0.6) between
+ id of Msgs object <pot> and Msgs in the dict <defs>.
+ Return value is the Msgs object in <defs> with highest ratio,
+ False is returned if no suitable Msgs is found.
+ '''
+ limit = 0.6
+ l, po = limit - 0.01, False
+ s = difflib.SequenceMatcher(lambda x: x == ' "', '', pot.get_clean_id())
+ len2 = len(pot.get_clean_id())
+ for candidate in defs.values():
+ if candidate.str == 'msgstr ""\n': # Empty translation
+ continue
+ if candidate.id == 'msgid ""\n': # Empty msgid (header)
+ continue
+ len1 = len(candidate.get_clean_id())
+ if len2 > 2 * len1 or len1 > 1.5 * len2: # Simple and fast tests first
+ continue
+ s.set_seq1(candidate.get_clean_id())
+ if s.quick_ratio() < l:
+ continue
+ r = s.ratio() # This is expensive
+ if r > l:
+ l, po = r, candidate
+ return po
+
+def flags(po, pot, fuzzy = False, obs = False):
+ '''
+ Create flag field from flag field in Msgs objects <po> and
+ <pot>. When <fuzzy> is true <po>\'s flags are ignored and the
+ fuzzy flag is added. If <obs> is set then most flags but fuzzy are
+ removed. If the global variable option.docstrings is set then
+ docstring flags will not be removed. The return value is a string
+ which holds the combined flag.
+ '''
+ global option
+ flag = ''
+ if po.flag or pot.flag or fuzzy:
+ if not fuzzy:
+ flag = '%s, %s' % (po.flag.strip(), pot.flag.strip())
+ else:
+ flag = '%s, %s' % ('#, fuzzy', pot.flag.strip())
+ flag = flag.split(', ')
+ fl = {}
+ flag = [fl.setdefault(f, f) for f in flag if f not in fl and f]
+ if not option.docstrings:
+ try:
+ flag.remove('docstring')
+ except ValueError:
+ pass
+ if obs:
+ removes = ['c-format', 'python-format', 'docstring']
+ for remove in removes:
+ try:
+ flag.remove(remove)
+ except ValueError:
+ pass
+ # Put fuzzy first
+ if 'fuzzy' in flag and not flag.index('fuzzy') == 1:
+ i = flag.index('fuzzy')
+ flag[1], flag[i] = flag[i], flag[1]
+
+ if len(flag) == 1:
+ flag = ''
+ else:
+ flag = ', '.join(flag) + '\n'
+ return flag
+
+def add(pot, po, fuzzy = False):
+ '''
+ Build a new entry from the Msgs objects <pot> and <pot>. If
+ <fuzzy> is true, <po>\'s flag field is ignored (in
+ flags()). Returns a multiline string with a up to date entry.
+ '''
+ msg = []
+ msg.append(po.cmt)
+ msg.append(pot.autocmt)
+ msg.append(pot.ref)
+ msg.append(flags(po, pot, fuzzy = fuzzy))
+ msg.append(pot.id)
+ msg.append(po.str)
+ return ''.join(msg)
+
+def header(pot, defs):
+ '''
+ Update date in header entry. Returns the updated header entry.
+ '''
+ try:
+ [po] = [ d for d in defs.values() if d.id == 'msgid ""\n' ]
+ except ValueError:
+ raise MsgmergeError('Error: did not find header in po file.')
+
+ r = re.compile(r'(.*^"POT-Creation-Date:\s+)(.*?)(\\n"$.*)',
+ re.MULTILINE | re.DOTALL)
+ m = r.match(pot.str)
+ if not m:
+ raise MsgmergeError(
+ 'Error: did not find POT-Creation-Date field in pot file.')
+
+ subs = '\\1%s\\3' % m.group(2)
+ _, count = r.subn(subs, po.str)
+ if not count == 1:
+ raise MsgmergeError(
+ 'Error: did not find POT-Creation-Date field in po file.')
+ return po
+
+def match(defs, refs):
+ '''
+ Try to match Msgs objects in <refs> with Msgs objects in
+ <defs>. The return value is a list with po entries.
+ '''
+ global option
+ matches = []
+ empty = Msgs('msgid ""\n', 'msgstr ""\n', '', -1, 'str')
+ deco = [(r.lno, r) for r in refs.values()]
+ deco.sort()
+ po = header(deco.pop(0)[1], defs) # Header entry
+ matches.append(add(empty, po))
+ po.used()
+ sorted = [ a[1] for a in deco ]
+ for pot in sorted:
+ if option.verbose:
+ sys.stderr.write('.')
+ po = defs.get(pot.id, False) # Perfect match
+ if po:
+ matches.append(add(pot, po))
+ po.used(); pot.used()
+ continue
+ po = fuzzy_match(pot, defs) # Fuzzy match
+ if po:
+ matches.append(add(pot, po, fuzzy = True))
+ po.used(); pot.used()
+ continue
+ matches.append(add(pot, empty)) # No match
+
+ obsolete(defs, matches)
+ return matches
+
+def obsolete(defs, matches):
+ '''Handle obsolete translations.'''
+ deco = [ (d.lno, d) for d in defs.values() if
+ d.count == 0 and not d.str == 'msgstr ""\n' ]
+ deco.sort()
+ empty = Msgs('msgid ""\n', 'msgstr ""\n', '', -1, 'str')
+ obs = [ o[1] for o in deco ]
+ for o in obs:
+ o.flag = flags(o, empty, obs = True)
+ o.obsolete()
+ matches.append('%s%s%s' % (o.flag, o.id, o.str))
+
+def help():
+ '''Print help text and exit.'''
+ print __doc__
+ sys.exit(0)
+
+def cmdline():
+ '''Parse options and arguments from command line.'''
+ advice = 'Try `%(name)s --help\' for more information.'
+ try:
+ long_opt = ['help', 'version', 'update', 'output-file=',
+ 'quiet', 'silent', 'docstrings', 'suffix', 'backup']
+ opts, args = getopt.getopt(sys.argv[1:], 'hVUo:qD', long_opt)
+ except getopt.error, msg:
+ print '%s: %s\n%s' % ('%(name)s', msg, advice) % globals()
+ sys.exit(1)
+
+ option = Options(cmdline = True)
+ for opt, arg in opts:
+ if opt in ['-h', '--help']:
+ help()
+ elif opt in ['-V', '--version']:
+ print '%(name)s %(__version__)s' % globals()
+ sys.exit(0)
+ elif opt in ['-o', '--output-file']:
+ option.outfile = arg
+ elif opt in ['-U', '--update']:
+ option.update = True
+ elif opt in ['-q', '--silent', '--quiet']:
+ option.verbose = False
+ elif opt in ['-D', '--docstrings']:
+ option.docstrings = True
+ elif opt in ['--suffix']:
+ option.suffix = arg
+ elif opt in ['--backup']:
+ option.backup = arg
+
+ # Sanity checks
+ warn = False
+ if option.update and option.outfile:
+ warn = '--update and --output-file are mutually exclusive.'
+ if len(args) == 0:
+ warn = 'no input files given.'
+ elif len(args) == 1 or len(args) > 2:
+ warn = 'exactly 2 input files required.'
+ if warn:
+ print '%s: %s\n%s' % ('%(name)s', warn, advice) % globals()
+ sys.exit(1)
+
+ if option.update:
+ option.outfile = args[0]
+ elif not option.outfile:
+ option.outfile = '-'
+
+ defs, refs = args
+
+ try:
+ merge(defs, refs, option = option)
+ except MsgmergeError, err:
+ print '%(name)s: ' % globals() + '%s' % err
+ sys.exit(1)
+
+def io(iofile, mode = 'rU'):
+ '''Wrapper around open().'''
+ try:
+ fo = open(iofile, mode)
+ if 'r' in mode and fo.read(3) != codecs.BOM_UTF8:
+ fo.seek(0)
+
+ except IOError, msg:
+ raise MsgmergeError('error while opening file: %s: %s.' %
+ (msg[1], iofile))
+ return fo
+
+def backup(infile):
+ '''Handle backup of files in update mode'''
+ os.environ.get('VERSION_CONTROL', '')
+ suffix = os.environ.get('SIMPLE_BACKUP_SUFFIX', '~')
+
+ backup_file = '%s%s' % (infile, suffix)
+
+def changes(new, old):
+ return cmp(''.join(old), '\n'.join(new))
+
+def write(matches, outfile):
+ '''Write the list <matches> to file <outfile>'''
+ if not outfile == '-':
+ fd = io(outfile, 'w')
+ else:
+ fd = sys.stdout
+ fd.write('\n'.join(matches))
+
+def merge(def_file, ref_file, update = True, outfile = '-',
+ docstrings = True, suffix = '~', backup = True,
+ verbose = True, **kwds):
+ '''
+ Merge po file <def_file> with pot file <ref_file> . If <update> is
+ set to True then only update if there are changes to the po
+ file. Set outfile to write updated po file to an another file. Set
+ to `-\' for writing to standard out. If docstrings is False
+ docstrings flag will removed. Set verbose to False to suppress
+ progress indicators. <kwds> is used to pass options from the
+ command line interface.
+ '''
+ global option
+ option = kwds.get('option', Options(update = update,
+ outfile = outfile,
+ docstrings = docstrings,
+ suffix = suffix,
+ backup = backup,
+ verbose = verbose))
+ def_msgs = parse(def_file, 'msgstr')
+ ref_msgs = parse(ref_file, 'msgid')
+ if verbose and not __name__ == '__main__':
+ print >> sys.stderr, 'Merging %s with %s' % (ref_file, def_file)
+ updated_lines = match(def_msgs, ref_msgs)
+ if option.verbose:
+ print >> sys.stderr, ' done.'
+ if not option.update:
+ write(updated_lines, option.outfile)
+ elif option.update and changes(updated_lines, io(def_file).readlines()):
+ write(updated_lines, def_file)
+
+def merge_dir(directory, pot = False, include = [], exclude = [],
+ verbose = True):
+ '''
+ Tries to merge a directory of po files. Uses simple glob to find
+ po files and pot file. The parameter <pot> can be used to specify
+ the pot file in the directory. If the list <include> is given only
+ files in this list is merged. Use the list <exclude> to exclude
+ files to be merged. This function is only useful if po files and
+ pot file are in the same directory. Set <verbose> to get
+ information when running.
+ '''
+ if directory[-1] == '/':
+ directory = os.path.dirname(directory)
+ if pot:
+ pot = os.path.basename(pot)
+ else:
+ pot = glob.glob('%s/*.pot' % directory)
+ if not pot:
+ raise MsgmergeError('No pot file found.')
+ elif len(pot) > 1:
+ raise MsgmergeError('More than one pot file found: %s.' % pot)
+ pot = os.path.basename(pot[0])
+
+ if not include:
+ pos = glob.glob('%s/*po' % directory)
+ if not len(pos) > 1:
+ raise MsgmergeError('No po file(s) found.')
+ pos = [ os.path.basename(po) for po in pos ]
+ else:
+ pos = [ os.path.basename(po) for po in include ]
+
+ for po in exclude:
+ try:
+ pos.remove(po)
+ except ValueError:
+ pass
+ format = '%s/%s'
+ for po in pos:
+ try:
+ merge(format % (directory, po), format % (directory, pot),
+ update = True, verbose = verbose,
+ outfile = format % (directory, po))
+ except MsgmergeError, err:
+ if verbose:
+ print >> sys.stderr, '%s Not updated.' % err
+ else:
+ print >> sys.stderr, '%s %s not updated.' % (err, po)
+
+if __name__ == '__main__':
+ cmdline()