summaryrefslogtreecommitdiffstatshomepage
path: root/scripts/build/msgmerge.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/build/msgmerge.py')
-rw-r--r--scripts/build/msgmerge.py675
1 files changed, 0 insertions, 675 deletions
diff --git a/scripts/build/msgmerge.py b/scripts/build/msgmerge.py
deleted file mode 100644
index 82fe3efdb97..00000000000
--- a/scripts/build/msgmerge.py
+++ /dev/null
@@ -1,675 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: iso-8859-1 -*-
-#
-# Copyright Terje Røsten <terjeros@phys.ntnu.no> Nov. 2003.
-#
-'''Merge two Uniforum style .po files together.
-
-This is a implementation (not complete) in Python of the GNU
-msgmerge(1) program. It can be used on the command line (or as a Python
-module).
-
-Usage: msgmerge.py [OPTIONS] def.po ref.pot
-
-The def.po file is an existing PO file with translations. The ref.pot
-file is the last created PO file with up-to-date source references but
-old translations, or a PO Template file.
-
-Options:
- -U, --update update def.po,
- do nothing if def.po is already up to date.
- -o, --output-file=FILE write output to file FILE. Output is written to
- stdout if set to - or if the option is not present.
- -D, --docstrings don\'t remove docstring flag.
- -h, --help display help text and exit.
- -V, --version display version and exit.
- -q, --quiet, --silent suppress progress indicators.
-'''
-from __future__ import generators
-
-if not __name__ == '__main__':
- __doc__ += '''\
-
-When used as module the interesting functions are merge() and
-merge_dir().
-
-The merge() function does the same as the command line version, and
-the arguments are as follows. The first argument is the def.po file,
-then the ref.pot file. The third argument controls whether do work in
-update mode or not, then the next argument sets the output file. Set
-the next argument to False to remove docstring flags. The last
-argument can be used to suppress progress indicators. The default is
-to work in update mode with progress indicators.
-
-Example:
- merge("def.po", "ref.pot")
- merge the files def.po and ref.pot and write output to def.po if
- there are any changes.
- merge("def.po", "red.pot", docstrings = False, verbose = False,
- update = False, outfile = "-")
- merge the files def.po and ref.pot and write output to stdout,
- remove docstring flag and be quiet.
-
-The merge_dir() function is useful when merging a directory of po
-files. The only required argument is the name of the directory with po
-files and the pot file. It will use simple glob to find the files. The
-second argument can be used to specify the pot file (in the
-directory). Third argument is a list of po files (then globbing will
-not be used) and the next argument is list of filename to exclude. The
-last argument can be used to suppress progress indicators. Docstring
-flag will not be removed.
-
-Example:
- merge_dir("po")
- merge (and update) all po files in directory po with the single pot
- file in the same directory.
-
-The module raises the MsgmergeError exception in case of error.
-'''
-__revision__ = '$Id: msgmerge.py,v 1.41 2003/11/18 19:10:42 terjeros Exp $'
-__version__ = '0.1'
-name = 'msgmerge.py'
-
-__all__ = [ 'merge', 'merge_dir', 'MsgmergeError' ]
-
-import sys
-import re
-import string
-import getopt
-import difflib
-import glob
-import os.path
-import codecs
-
-try:
- True, False
-except NameError:
- True, False = 1, 0
-
-class Msgs:
- '''Class to hold information about messages.'''
- width = 80
- file = ''
- def __init__(self, msgid, msgstr, flag, lno, entry, **kwds):
- self.id = msgid
- self.str = msgstr
- self.cmt = kwds.get('cmt', '')
- self.ref = kwds.get('ref', '')
- self.autocmt = kwds.get('autocmt', '')
- self.flag = flag
- self.entry = entry
- self.lno = lno
- self.count = 0
- def wash(self):
- self.id = wash(self.id, width = self.width,
- filename = self.file, lno = self.lno)
- self.str = wash(self.str, 'msgstr', width = self.width,
- filename = self.file, lno = self.lno)
- def used(self):
- self.count += 1
- def get_clean_id(self):
- return self.id.replace('msgid "','', 1)
- def obsolete(self):
- self.width -= len('#~ ')
- self.wash()
- t = [ '#~ %s\n' % s for s in self.id.splitlines() ]
- self.id = ''.join(t)
- t = [ '#~ %s\n' % s for s in self.str.splitlines() ]
- self.str = ''.join(t)
-
-class Options:
- '''Class to hold options'''
- def __init__(self, cmdline = False, **kwds):
- if not cmdline:
- self.update = kwds.get('update', True)
- self.outfile = kwds.get('outfile', '-')
- self.docstrings = kwds.get('docstrings', True)
- self.verbose = kwds.get('verbose', False)
- self.suffix = kwds.get('suffix', '~')
- self.backup = kwds.get('backup', True)
- else:
- self.update = False
- self.outfile = False
- self.docstrings = False
- self.verbose = True
- self.suffix = '~'
- self.backup = True
-
-class MsgmergeError(Exception):
- '''Exception class for msgmerge'''
-
-def gen(lines):
- '''
- Generator which returns a line (with the obsolete prefix removed)
- from the list of lines in <lines>, the line number is also
- returned.
- '''
- lno = 0
- for l in lines:
- lno += 1
- yield l.replace('#~ ', '', 1), lno
- yield l, lno
-
-def slurp(s, g, sign):
- '''
- The string returned from iterator <g>\'s next() method is added to
- the string <s> if string returned is beginning with the string
- <sign>. The return value is the first returned string which do not
- start with <sign>, the line number, the iterator <g> and the
- (possibly) updated string <s>.
- '''
- l, lno = g.next()
- while l.startswith(sign) or (sign == '# ' and l.strip() == '#'):
- s += l
- l, lno = g.next()
- return l, lno, g, s
-
-def splitted_fit(chunk, line, width, break_always, break_after_space):
- '''
- Check if string <chunk> can be splitted by newline to fit into
- string <line> with width smaller than <width>. The return value is
- a tuple where the first element is the part of chunk which fits
- and the second element is the rest of chunk.
- '''
- ret = '', chunk
- l = len(chunk)
- for i in range(l - 1, -1, -1):
- if chunk[i] in break_always and len(chunk[0:i] + line) <= width:
- ret = chunk[0:i], chunk[i:]
- break
- elif chunk[i] in break_after_space and i and chunk[i-1].strip() == '':
- ret = chunk[0:i], chunk[i:]
- break
- elif chunk[i] == '\\' and len(chunk[i:]) > 1 and chunk[i+1] == '"' \
- and len(chunk[0:i] + line) <= width:
- ret = chunk[0:i], chunk[i:]
- break
- return ret
-
-def wrap(msg, width):
- '''
- Accept a list <msg> of strings to wrap, each string is wrapped to
- width <width> and surrounded with a pair of ". The return value is
- a string with these wrapped strings joined together with newlines.
- '''
- if msg.isspace() or not msg:
- return '"%s"' % msg
-
- # \ and " is here, but " is special in po files.
- break_always = '$%+({['
- # XXX what about: « © » ¦ § etc?
- break_after_space = '_-=^`~\'<|>&*#@'
- enders = '.:,;!?/])}|%-'
- extra = string.punctuation
- for c in enders:
- extra = extra.replace(c, '')
- escaped = { 'enders' : re.escape(enders),
- 'extra' : re.escape(extra) }
- regex = r'([\w%(extra)s]*[\s%(enders)s)]+[\s%(enders)s]*)' % escaped
- r = re.compile(regex, re.UNICODE)
- msg = [ m for m in r.split(msg) if not m == '']
-
- lines = []
- line = msg.pop(0)
-
- # Handle \n on end of line
- if len(msg) > 1 and msg[-1] == 'n' and len(msg[-2]) > 0 \
- and msg[-2][-1] == '\\':
- msg[-2] += msg[-1]
- msg.pop()
- # Do not allow a single \n on a line
- if len(msg) > 2 and msg[-1] == '\\n':
- msg[-2] += msg[-1]
- msg.pop()
-
- for m in msg:
- if len(line) > width or len(m) > width or len(line + m) > width:
- fit, rest = splitted_fit(m, line, width, break_always,
- break_after_space)
- line += fit
- lines.append(line)
- line = rest
- else:
- line += m
- lines.append(line)
- lines = [ '"%s"' % l for l in lines ]
- return '\n'.join(lines)
-
-def normalize(lines):
- '''
- Normalize <lines>: e.g "\n\nText\n\n" becomes:
- "\n"
- "\n"
- "Text\n"
- "\n"
- '''
- if 0 < lines.find('\\n') < len(lines) - 3:
- if lines[-3:] == '\\n"':
- lines = lines[:-3].replace('\\n','\\n"\n"').replace('""\n','') \
- + '\\n"'
- else:
- lines = lines.replace('\\n','\\n"\n"').replace('""\n','')
- return lines
-
-def wash(msg, idx = 'msgid', width = 80, **kwds):
- '''
- Do washing on the msgstr or msgid fields. Wrap the text to fit in
- width <width>. <msg> is a list of lines that makes up the field.
- <idx> indicate msgid or msgstr, <width> holds the width. <filename>
- and <lno> (line number) is picked up from <kwds>.
- Returns the washed field as a string.
- '''
- msg = normalize(msg)
- lines = msg.splitlines()
- size = len(lines)
- if size > 1 or len(msg) > width:
- washed = []
- # The first line is special
- m = re.match('^%s "(.*)"$' % (idx, ), lines[0])
- if not m:
- print lines[0]
- kwds['lno'] -= size + 1
- raise MsgmergeError('parse error: %(filename)s:%(lno)s.'
- % kwds)
- washed.append(m.group(1))
- if m.group(1).endswith(r'\n'):
- washed.append('')
- i = 0
- for line in lines[1:]:
- m = re.match('^"(\s*.*)"$', line)
- i += 1
- if not m:
- print line
- kwds['lno'] -= size - i + 1
- raise MsgmergeError('parse error: %(filename)s:%(lno)s.'
- % kwds)
- washed[-1] += m.group(1)
- if m.group(1).endswith(r'\n'):
- washed.append('')
- if washed[0] == '':
- washed.pop(0)
- if washed[-1] == '':
- washed.pop()
-
- washed = [ wrap(w, width - 3) for w in washed ] # " and \n removed.
-
- # One line or multiline
- if len(washed) == 1 and len('%s %s\n' % (idx, washed[0])) < width:
- washed = '%s %s\n' % (idx, washed[0])
- else:
- washed = '%s ""\n%s\n' % (idx, '\n'.join(washed))
- else:
- washed = msg
-
- return washed
-
-def parse(filename, entry):
- '''
- Parse po or pot file with name <filename>. Set the variable
- <entry> to msgid/msgstr to indicate pot/po file. The return value
- is a dict with msgid (washed) as key and Msgs instances as
- values.
- '''
- lines = io(filename).readlines()
- Msgs.file = filename
- messages = {}
- last = len(lines)
- g = gen(lines)
- cmt = autocmt = ref = flag = ''
- msgid = False
- lno = 0
- while not lno == last:
- l, lno = g.next()
- if l.startswith('# '):
- l, lno, g, cmt = slurp(l, g, '# ')
- if l.startswith('#.'):
- l, lno, g, autocmt = slurp(l, g, '#.')
- if l.startswith('#:'):
- l, lno, g, ref = slurp(l, g, '#:')
- if l.startswith('#,'):
- l, lno, g, flag = slurp(l, g, '#,')
- if l.startswith('msgid'):
- l, lno, g, msgid = slurp(l, g, '"')
- if l.startswith('msgstr'):
- l, lno, g, msgstr = slurp(l, g, '"')
-
- if not lno == last and not l.strip() == '':
- raise MsgmergeError('parse error: %s:%s.' % (filename, lno))
-
- if msgid and entry == 'msgstr':
- idx = wash(msgid, filename = filename, lno = lno)
- messages[idx] = Msgs(msgid, msgstr, flag, lno, entry, cmt = cmt)
- msgid = False; msgstr = cmt = autocmt = ref = flag = ''
- elif msgid and entry == 'msgid':
- idx = wash(msgid, filename = filename, lno = lno)
- messages[idx] = Msgs(msgid, msgstr, flag, lno, entry,
- autocmt = autocmt, ref = ref)
- msgid = False; msgstr = cmt = autocmt = ref = flag = ''
-
- for m in messages.values():
- m.wash()
- return messages
-
-def fuzzy_match(pot, defs):
- '''
- Try to find the best difflib match (with ratio > 0.6) between
- id of Msgs object <pot> and Msgs in the dict <defs>.
- Return value is the Msgs object in <defs> with highest ratio,
- False is returned if no suitable Msgs is found.
- '''
- limit = 0.6
- l, po = limit - 0.01, False
- s = difflib.SequenceMatcher(lambda x: x == ' "', '', pot.get_clean_id())
- len2 = len(pot.get_clean_id())
- for candidate in defs.values():
- if candidate.str == 'msgstr ""\n': # Empty translation
- continue
- if candidate.id == 'msgid ""\n': # Empty msgid (header)
- continue
- len1 = len(candidate.get_clean_id())
- if len2 > 2 * len1 or len1 > 1.5 * len2: # Simple and fast tests first
- continue
- s.set_seq1(candidate.get_clean_id())
- if s.quick_ratio() < l:
- continue
- r = s.ratio() # This is expensive
- if r > l:
- l, po = r, candidate
- return po
-
-def flags(po, pot, fuzzy = False, obs = False):
- '''
- Create flag field from flag field in Msgs objects <po> and
- <pot>. When <fuzzy> is true <po>\'s flags are ignored and the
- fuzzy flag is added. If <obs> is set then most flags but fuzzy are
- removed. If the global variable option.docstrings is set then
- docstring flags will not be removed. The return value is a string
- which holds the combined flag.
- '''
- global option
- flag = ''
- if po.flag or pot.flag or fuzzy:
- if not fuzzy:
- flag = '%s, %s' % (po.flag.strip(), pot.flag.strip())
- else:
- flag = '%s, %s' % ('#, fuzzy', pot.flag.strip())
- flag = flag.split(', ')
- fl = {}
- flag = [fl.setdefault(f, f) for f in flag if f not in fl and f]
- if not option.docstrings:
- try:
- flag.remove('docstring')
- except ValueError:
- pass
- if obs:
- removes = ['c-format', 'python-format', 'docstring']
- for remove in removes:
- try:
- flag.remove(remove)
- except ValueError:
- pass
- # Put fuzzy first
- if 'fuzzy' in flag and not flag.index('fuzzy') == 1:
- i = flag.index('fuzzy')
- flag[1], flag[i] = flag[i], flag[1]
-
- if len(flag) == 1:
- flag = ''
- else:
- flag = ', '.join(flag) + '\n'
- return flag
-
-def add(pot, po, fuzzy = False):
- '''
- Build a new entry from the Msgs objects <pot> and <pot>. If
- <fuzzy> is true, <po>\'s flag field is ignored (in
- flags()). Returns a multiline string with a up to date entry.
- '''
- msg = []
- msg.append(po.cmt)
- msg.append(pot.autocmt)
- msg.append(pot.ref)
- msg.append(flags(po, pot, fuzzy = fuzzy))
- msg.append(pot.id)
- msg.append(po.str)
- return ''.join(msg)
-
-def header(pot, defs):
- '''
- Update date in header entry. Returns the updated header entry.
- '''
- try:
- [po] = [ d for d in defs.values() if d.id == 'msgid ""\n' ]
- except ValueError:
- raise MsgmergeError('Error: did not find header in po file.')
-
- r = re.compile(r'(.*^"POT-Creation-Date:\s+)(.*?)(\\n"$.*)',
- re.MULTILINE | re.DOTALL)
- m = r.match(pot.str)
- if not m:
- raise MsgmergeError(
- 'Error: did not find POT-Creation-Date field in pot file.')
-
- subs = '\\1%s\\3' % m.group(2)
- _, count = r.subn(subs, po.str)
- if not count == 1:
- raise MsgmergeError(
- 'Error: did not find POT-Creation-Date field in po file.')
- return po
-
-def match(defs, refs):
- '''
- Try to match Msgs objects in <refs> with Msgs objects in
- <defs>. The return value is a list with po entries.
- '''
- global option
- matches = []
- empty = Msgs('msgid ""\n', 'msgstr ""\n', '', -1, 'str')
- deco = [(r.lno, r) for r in refs.values()]
- deco.sort()
- po = header(deco.pop(0)[1], defs) # Header entry
- matches.append(add(empty, po))
- po.used()
- sorted = [ a[1] for a in deco ]
- for pot in sorted:
- if option.verbose:
- sys.stderr.write('.')
- po = defs.get(pot.id, False) # Perfect match
- if po:
- matches.append(add(pot, po))
- po.used(); pot.used()
- continue
- po = fuzzy_match(pot, defs) # Fuzzy match
- if po:
- matches.append(add(pot, po, fuzzy = True))
- po.used(); pot.used()
- continue
- matches.append(add(pot, empty)) # No match
-
- obsolete(defs, matches)
- return matches
-
-def obsolete(defs, matches):
- '''Handle obsolete translations.'''
- deco = [ (d.lno, d) for d in defs.values() if
- d.count == 0 and not d.str == 'msgstr ""\n' ]
- deco.sort()
- empty = Msgs('msgid ""\n', 'msgstr ""\n', '', -1, 'str')
- obs = [ o[1] for o in deco ]
- for o in obs:
- o.flag = flags(o, empty, obs = True)
- o.obsolete()
- matches.append('%s%s%s' % (o.flag, o.id, o.str))
-
-def help():
- '''Print help text and exit.'''
- print __doc__
- sys.exit(0)
-
-def cmdline():
- '''Parse options and arguments from command line.'''
- advice = 'Try `%(name)s --help\' for more information.'
- try:
- long_opt = ['help', 'version', 'update', 'output-file=',
- 'quiet', 'silent', 'docstrings', 'suffix', 'backup']
- opts, args = getopt.getopt(sys.argv[1:], 'hVUo:qD', long_opt)
- except getopt.error, msg:
- print '%s: %s\n%s' % ('%(name)s', msg, advice) % globals()
- sys.exit(1)
-
- option = Options(cmdline = True)
- for opt, arg in opts:
- if opt in ['-h', '--help']:
- help()
- elif opt in ['-V', '--version']:
- print '%(name)s %(__version__)s' % globals()
- sys.exit(0)
- elif opt in ['-o', '--output-file']:
- option.outfile = arg
- elif opt in ['-U', '--update']:
- option.update = True
- elif opt in ['-q', '--silent', '--quiet']:
- option.verbose = False
- elif opt in ['-D', '--docstrings']:
- option.docstrings = True
- elif opt in ['--suffix']:
- option.suffix = arg
- elif opt in ['--backup']:
- option.backup = arg
-
- # Sanity checks
- warn = False
- if option.update and option.outfile:
- warn = '--update and --output-file are mutually exclusive.'
- if len(args) == 0:
- warn = 'no input files given.'
- elif len(args) == 1 or len(args) > 2:
- warn = 'exactly 2 input files required.'
- if warn:
- print '%s: %s\n%s' % ('%(name)s', warn, advice) % globals()
- sys.exit(1)
-
- if option.update:
- option.outfile = args[0]
- elif not option.outfile:
- option.outfile = '-'
-
- defs, refs = args
-
- try:
- merge(defs, refs, option = option)
- except MsgmergeError, err:
- print '%(name)s: ' % globals() + '%s' % err
- sys.exit(1)
-
-def io(iofile, mode = 'rU'):
- '''Wrapper around open().'''
- try:
- fo = open(iofile, mode)
- if 'r' in mode and fo.read(3) != codecs.BOM_UTF8:
- fo.seek(0)
-
- except IOError, msg:
- raise MsgmergeError('error while opening file: %s: %s.' %
- (msg[1], iofile))
- return fo
-
-def backup(infile):
- '''Handle backup of files in update mode'''
- os.environ.get('VERSION_CONTROL', '')
- suffix = os.environ.get('SIMPLE_BACKUP_SUFFIX', '~')
-
- backup_file = '%s%s' % (infile, suffix)
-
-def changes(new, old):
- return cmp(''.join(old), '\n'.join(new))
-
-def write(matches, outfile):
- '''Write the list <matches> to file <outfile>'''
- if not outfile == '-':
- fd = io(outfile, 'w')
- else:
- fd = sys.stdout
- fd.write('\n'.join(matches))
-
-def merge(def_file, ref_file, update = True, outfile = '-',
- docstrings = True, suffix = '~', backup = True,
- verbose = True, **kwds):
- '''
- Merge po file <def_file> with pot file <ref_file> . If <update> is
- set to True then only update if there are changes to the po
- file. Set outfile to write updated po file to an another file. Set
- to `-\' for writing to standard out. If docstrings is False
- docstrings flag will removed. Set verbose to False to suppress
- progress indicators. <kwds> is used to pass options from the
- command line interface.
- '''
- global option
- option = kwds.get('option', Options(update = update,
- outfile = outfile,
- docstrings = docstrings,
- suffix = suffix,
- backup = backup,
- verbose = verbose))
- def_msgs = parse(def_file, 'msgstr')
- ref_msgs = parse(ref_file, 'msgid')
- if verbose and not __name__ == '__main__':
- print >> sys.stderr, 'Merging %s with %s' % (ref_file, def_file)
- updated_lines = match(def_msgs, ref_msgs)
- if option.verbose:
- print >> sys.stderr, ' done.'
- if not option.update:
- write(updated_lines, option.outfile)
- elif option.update and changes(updated_lines, io(def_file).readlines()):
- write(updated_lines, def_file)
-
-def merge_dir(directory, pot = False, include = [], exclude = [],
- verbose = True):
- '''
- Tries to merge a directory of po files. Uses simple glob to find
- po files and pot file. The parameter <pot> can be used to specify
- the pot file in the directory. If the list <include> is given only
- files in this list is merged. Use the list <exclude> to exclude
- files to be merged. This function is only useful if po files and
- pot file are in the same directory. Set <verbose> to get
- information when running.
- '''
- if directory[-1] == '/':
- directory = os.path.dirname(directory)
- if pot:
- pot = os.path.basename(pot)
- else:
- pot = glob.glob('%s/*.pot' % directory)
- if not pot:
- raise MsgmergeError('No pot file found.')
- elif len(pot) > 1:
- raise MsgmergeError('More than one pot file found: %s.' % pot)
- pot = os.path.basename(pot[0])
-
- if not include:
- pos = glob.glob('%s/*po' % directory)
- if not len(pos) > 1:
- raise MsgmergeError('No po file(s) found.')
- pos = [ os.path.basename(po) for po in pos ]
- else:
- pos = [ os.path.basename(po) for po in include ]
-
- for po in exclude:
- try:
- pos.remove(po)
- except ValueError:
- pass
- format = '%s/%s'
- for po in pos:
- try:
- merge(format % (directory, po), format % (directory, pot),
- update = True, verbose = verbose,
- outfile = format % (directory, po))
- except MsgmergeError, err:
- if verbose:
- print >> sys.stderr, '%s Not updated.' % err
- else:
- print >> sys.stderr, '%s %s not updated.' % (err, po)
-
-if __name__ == '__main__':
- cmdline()