diff options
Diffstat (limited to 'docs/release/scripts/build/msgfmt.py')
-rw-r--r-- | docs/release/scripts/build/msgfmt.py | 155 |
1 files changed, 110 insertions, 45 deletions
diff --git a/docs/release/scripts/build/msgfmt.py b/docs/release/scripts/build/msgfmt.py index f28c2fcf6c6..deb02ae32b2 100644 --- a/docs/release/scripts/build/msgfmt.py +++ b/docs/release/scripts/build/msgfmt.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python2 -# -*- coding: iso-8859-1 -*- -# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de> +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de> """Generate binary message catalog from textual translation description. @@ -25,49 +25,71 @@ Options: Display version information and exit. """ +from __future__ import print_function import os import sys -import ast import getopt import struct import array +import re +import codecs +from email.parser import HeaderParser -__version__ = "1.1" +__version__ = "1.2" MESSAGES = {} - + def usage(code, msg=''): - print >> sys.stderr, __doc__ + print(__doc__, file=sys.stderr) if msg: - print >> sys.stderr, msg + print(msg, file=sys.stderr) sys.exit(code) - + def add(id, str, fuzzy): "Add a non-fuzzy translation to the dictionary." global MESSAGES if not fuzzy and str: MESSAGES[id] = str +def dequote(s): + if (s[0] == s[-1]) and s.startswith(("'", '"')): + return s[1:-1] + return s + +# decode_escapes from http://stackoverflow.com/a/24519338 +ESCAPE_SEQUENCE_RE = re.compile(r''' + ( \\U........ # 8-digit hex escapes + | \\u.... # 4-digit hex escapes + | \\x.. # 2-digit hex escapes + | \\[0-7]{1,3} # Octal escapes + | \\N\{[^}]+\} # Unicode characters by name + | \\[\\'"abfnrtv] # Single-character escapes + )''', re.UNICODE | re.VERBOSE) + +def decode_escapes(s): + def decode_match(match): + return codecs.decode(match.group(0), 'unicode-escape') + + return ESCAPE_SEQUENCE_RE.sub(decode_match, s) + - def generate(): "Return the generated output." global MESSAGES - keys = MESSAGES.keys() # the keys are sorted in the .mo file - keys.sort() + keys = sorted(MESSAGES.keys()) offsets = [] - ids = strs = '' + ids = strs = b'' for id in keys: # For each string, we need size and file offset. Each string is NUL # terminated; the NUL does not count into the size. offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id]))) - ids += id + '\0' - strs += MESSAGES[id] + '\0' + ids += id + b'\0' + strs += MESSAGES[id] + b'\0' output = '' # The header is 7 32-bit unsigned integers. We don't use hash tables, so # the keys start right after the index tables. @@ -84,7 +106,7 @@ def generate(): voffsets += [l2, o2+valuestart] offsets = koffsets + voffsets output = struct.pack("Iiiiiii", - 0x950412deL, # Magic + 0x950412de, # Magic 0, # Version len(keys), # # of entries 7*4, # start of key index @@ -96,7 +118,7 @@ def generate(): return output - + def make(filename, outfile): ID = 1 STR = 2 @@ -110,18 +132,27 @@ def make(filename, outfile): outfile = os.path.splitext(infile)[0] + '.mo' try: - lines = open(infile).readlines() - except IOError, msg: - print >> sys.stderr, msg + lines = open(infile, 'rb').readlines() + except IOError as msg: + print(msg, file=sys.stderr) sys.exit(1) section = None fuzzy = 0 + empty = 0 + header_attempted = False + + # Start off assuming Latin-1, so everything decodes without failure, + # until we know the exact encoding + encoding = 'latin-1' + + # Start off assuming Latin-1, so everything decodes without failure, + # until we know the exact encoding + encoding = 'latin-1' # Parse the catalog - lno = 0 - for l in lines: - lno += 1 + for lno, l in enumerate(lines): + l = l.decode(encoding) # If we get a comment line after a msgstr, this is a new entry if l[0] == '#' and section == STR: add(msgid, msgstr, fuzzy) @@ -137,49 +168,83 @@ def make(filename, outfile): if l.startswith('msgid') and not l.startswith('msgid_plural'): if section == STR: add(msgid, msgstr, fuzzy) + if not msgid: + # See whether there is an encoding declaration + p = HeaderParser() + charset = p.parsestr(msgstr.decode(encoding)).get_content_charset() + if charset: + encoding = charset section = ID l = l[5:] - msgid = msgstr = '' + msgid = msgstr = b'' is_plural = False + if l.strip() == '""': + # Check if next line is msgstr. If so, this is a multiline msgid. + if lines[lno+1].decode(encoding).startswith('msgstr'): + # If this is the first empty msgid and is followed by msgstr, this is the header, which may contain the encoding declaration. + # Otherwise this file is not valid + if empty > 1: + print("Found multiple empty msgids on line " + str(lno) + ", not valid!") + empty += 1 # This is a message with plural forms elif l.startswith('msgid_plural'): if section != ID: - print >> sys.stderr, 'msgid_plural not preceded by msgid on %s:%d' %\ - (infile, lno) + print('msgid_plural not preceded by msgid on %s:%d' % (infile, lno), + file=sys.stderr) sys.exit(1) l = l[12:] - msgid += '\0' # separator of singular and plural + msgid += b'\0' # separator of singular and plural is_plural = True # Now we are in a msgstr section elif l.startswith('msgstr'): section = STR if l.startswith('msgstr['): if not is_plural: - print >> sys.stderr, 'plural without msgid_plural on %s:%d' %\ - (infile, lno) + print('plural without msgid_plural on %s:%d' % (infile, lno), + file=sys.stderr) sys.exit(1) l = l.split(']', 1)[1] if msgstr: - msgstr += '\0' # Separator of the various plural forms + msgstr += b'\0' # Separator of the various plural forms else: + if (l[6:].strip() == '""') and (empty == 1) and (not header_attempted): + header = "" + # parse up until next empty line = end of header + hdrno = lno + while(hdrno < len(lines)-1): + # This is a roundabout way to strip non-ASCII unicode characters from the header. + # As we are only parsing out the encoding, we don't need any unicode chars in it. + l = lines[hdrno+1].decode('unicode_escape').encode('ascii','ignore').decode(encoding) + if l.strip(): + header += decode_escapes(dequote(l.strip())) + else: + break + hdrno += 1 + # See whether there is an encoding declaration + if(hdrno > lno): + p = HeaderParser() + charset = p.parsestr(str(header)).get_content_charset() + header_attempted = True + if charset: + encoding = charset if is_plural: - print >> sys.stderr, 'indexed msgstr required for plural on %s:%d' %\ - (infile, lno) + print('indexed msgstr required for plural on %s:%d' % (infile, lno), + file=sys.stderr) sys.exit(1) l = l[6:] # Skip empty lines l = l.strip() if not l: continue - l = ast.literal_eval(l) + l = decode_escapes(dequote(l)) # strip quotes and replace newlines if present if section == ID: - msgid += l + msgid += l.encode(encoding) elif section == STR: - msgstr += l + msgstr += l.encode(encoding) else: - print >> sys.stderr, 'Syntax error on %s:%d' % (infile, lno), \ - 'before:' - print >> sys.stderr, l + print('Syntax error on %s:%d' % (infile, lno), \ + 'before:', file=sys.stderr) + print(l, file=sys.stderr) sys.exit(1) # Add last entry if section == STR: @@ -190,16 +255,16 @@ def make(filename, outfile): try: open(outfile,"wb").write(output) - except IOError,msg: - print >> sys.stderr, msg + except IOError as msg: + print(msg, file=sys.stderr) + - def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'hVo:', ['help', 'version', 'output-file=']) - except getopt.error, msg: + except getopt.error as msg: usage(1, msg) outfile = None @@ -208,14 +273,14 @@ def main(): if opt in ('-h', '--help'): usage(0) elif opt in ('-V', '--version'): - print >> sys.stderr, "msgfmt.py", __version__ + print("msgfmt.py", __version__) sys.exit(0) elif opt in ('-o', '--output-file'): outfile = arg # do it if not args: - print >> sys.stderr, 'No input file given' - print >> sys.stderr, "Try `msgfmt --help' for more information." + print('No input file given', file=sys.stderr) + print("Try `msgfmt --help' for more information.", file=sys.stderr) return for filename in args: |