diff options
author | 2017-02-05 15:45:26 +0100 | |
---|---|---|
committer | 2017-02-05 15:46:07 +0100 | |
commit | 02c97cd0f7c4d630f1a90f9f2626746e47a28c09 (patch) | |
tree | 2b48d051a55f2b65525824546cd9fcf9e05907cf /3rdparty/catch/scripts/approvalTests.py | |
parent | d20b4ba6864b45d0c068d98d43e63c8bc5014a3b (diff) |
Updated Catch to latest (nw)
Diffstat (limited to '3rdparty/catch/scripts/approvalTests.py')
-rw-r--r-- | 3rdparty/catch/scripts/approvalTests.py | 221 |
1 files changed, 129 insertions, 92 deletions
diff --git a/3rdparty/catch/scripts/approvalTests.py b/3rdparty/catch/scripts/approvalTests.py index 618812f888d..6c51aa4eff2 100644 --- a/3rdparty/catch/scripts/approvalTests.py +++ b/3rdparty/catch/scripts/approvalTests.py @@ -1,115 +1,152 @@ -from __future__ import print_function +#!/usr/bin/env python + +from __future__ import print_function import os import sys import subprocess import re +import difflib +import scriptCommon from scriptCommon import catchPath -rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' ) +rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines') + -filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' ) -filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' ) -pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' ) -lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' ) -hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' ) -durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' ) -versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' ) -devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' ) +filelocParser = re.compile(r''' + .*/ + (.+\.[ch]pp) # filename + (?::|\() # : is starting separator between filename and line number on Linux, ( on Windows + ([0-9]*) # line number + \)? # Windows also has an ending separator, ) +''', re.VERBOSE) +lineNumberParser = re.compile(r' line="[0-9]*"') +hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b') +durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"') +timestampsParser = re.compile(r' timestamp="\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z"') +versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?') +nullParser = re.compile(r'\b(__null|nullptr)\b') +exeNameParser = re.compile(r''' + \b + (CatchSelfTest|SelfTest) # Expected executable name + (?:.exe)? # Executable name contains .exe on Windows. + \b +''', re.VERBOSE) +# This is a hack until something more reasonable is figured out +specialCaseParser = re.compile(r'file\((\d+)\)') if len(sys.argv) == 2: - cmdPath = sys.argv[1] + cmdPath = sys.argv[1] else: - cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' ) + cmdPath = os.path.join(catchPath, scriptCommon.getBuildExecutable()) overallResult = 0 -def filterLine( line ): - m = filenameParser.match( line ) - if m: - line = m.group(2) + m.group(3) - m2 = filelineParser.match( line ) - if m2: - line = m2.group(1) + "<line number>" + m2.group(3) - else: - m2 = lineNumberParser.match( line ) - if m2: - line = m2.group(1) + m2.group(2) - m = pathParser.match( line ) - if m: - path = "/" + m.group(2) - if path.startswith( catchPath ): - path = path[1+len(catchPath):] - line = m.group(1) + path + m.group(3) - m = devVersionParser.match( line ) - if m: - line = m.group(1) + "<version>" + m.group(2) - else: - m = versionParser.match( line ) - if m: - line = m.group(1) + "<version>" + m.group(2) - - while True: - m = hexParser.match( line ) - if m: - line = m.group(1) + "0x<hex digits>" + m.group(3) - else: - break - m = durationsParser.match( line ) - if m: - line = m.group(1) + 'time="{duration}"' + m.group(2) - return line - -def approve( baseName, args ): - global overallResult - args[0:0] = [cmdPath] - if not os.path.exists( cmdPath ): - raise Exception( "Executable doesn't exist at " + cmdPath ) - baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) ) - rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) - filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) ) - - f = open( rawResultsPath, 'w' ) - subprocess.call( args, stdout=f, stderr=f ) - f.close() - - rawFile = open( rawResultsPath, 'r' ) - filteredFile = open( filteredResultsPath, 'w' ) - for line in rawFile: - filteredFile.write( filterLine( line ).rstrip() + "\n" ) - filteredFile.close() - rawFile.close() - - os.remove( rawResultsPath ) - print() - print( baseName + ":" ) - if os.path.exists( baselinesPath ): - diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] ) - if diffResult == 0: - os.remove( filteredResultsPath ) - print( " \033[92mResults matched" ) - else: - print( " \n****************************\n \033[91mResults differed" ) - if diffResult > overallResult: - overallResult = diffResult - print( "\033[0m" ) - else: - print( " first approval" ) - if overallResult == 0: - overallResult = 1 +def diffFiles(fileA, fileB): + with open(fileA, 'r') as file: + aLines = file.readlines() + with open(fileB, 'r') as file: + bLines = file.readlines() + + shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1] + shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1] + + diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0) + return [line for line in diff if line[0] in ('+', '-')] + + +def filterLine(line): + if catchPath in line: + # make paths relative to Catch root + line = line.replace(catchPath + os.sep, '') + # go from \ in windows paths to / + line = line.replace('\\', '/') + + + # strip source line numbers + m = filelocParser.match(line) + if m: + # note that this also strips directories, leaving only the filename + filename, lnum = m.groups() + lnum = ":<line number>" if lnum else "" + line = filename + lnum + line[m.end():] + else: + line = lineNumberParser.sub(" ", line) + + # strip Catch version number + line = versionParser.sub("<version>", line) + + # replace *null* with 0 + line = nullParser.sub("0", line) + + # strip executable name + line = exeNameParser.sub("<exe-name>", line) + + # strip hexadecimal numbers (presumably pointer values) + line = hexParser.sub("0x<hex digits>", line) + + # strip durations and timestamps + line = durationsParser.sub(' time="{duration}"', line) + line = timestampsParser.sub(' timestamp="{iso8601-timestamp}"', line) + line = specialCaseParser.sub('file:\g<1>', line) + return line + + +def approve(baseName, args): + global overallResult + args[0:0] = [cmdPath] + if not os.path.exists(cmdPath): + raise Exception("Executable doesn't exist at " + cmdPath) + baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName)) + rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName)) + filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName)) + + f = open(rawResultsPath, 'w') + subprocess.call(args, stdout=f, stderr=f) + f.close() + + rawFile = open(rawResultsPath, 'r') + filteredFile = open(filteredResultsPath, 'w') + for line in rawFile: + filteredFile.write(filterLine(line).rstrip() + "\n") + filteredFile.close() + rawFile.close() + + os.remove(rawResultsPath) + print() + print(baseName + ":") + if os.path.exists(baselinesPath): + diffResult = diffFiles(baselinesPath, filteredResultsPath) + if diffResult: + print(''.join(diffResult)) + print(" \n****************************\n \033[91mResults differed") + if len(diffResult) > overallResult: + overallResult = len(diffResult) + else: + os.remove(filteredResultsPath) + print(" \033[92mResults matched") + print("\033[0m") + else: + print(" first approval") + if overallResult == 0: + overallResult = 1 + + +print("Running approvals against executable:") +print(" " + cmdPath) # Standard console reporter -approve( "console.std", ["~_"] ) +approve("console.std", ["~[c++11]~[!nonportable]", "--order", "lex"]) # console reporter, include passes, warn about No Assertions -approve( "console.sw", ["~_", "-s", "-w", "NoAssertions"] ) +approve("console.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"]) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4"] ) +approve("console.swa4", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) # junit reporter, include passes, warn about No Assertions -approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit"] ) +approve("junit.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) # xml reporter, include passes, warn about No Assertions -approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] ) +approve("xml.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) if overallResult != 0: - print( "run approve.py to approve new baselines" ) -exit( overallResult) + print("If these differenecs are expected run approve.py to approve new baselines") +exit(overallResult) |